def clean_requires_python(candidates):
"""Get a cleaned list of all the candidates with valid specifiers in the
`requires_python` attributes."""
all_candidates = []
sys_version = ".".join(map(str, sys.version_info[:3]))
from pipenv.vendor.packaging.version import parse as parse_version
py_version = parse_version(
os.environ.get("PIP_PYTHON_VERSION", sys_version))
for c in candidates:
requires_python = _get_requires_python(c)
if requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if requires_python.isdigit():
requires_python = ">={0},<{1}".format(requires_python,
int(requires_python) + 1)
try:
specifierset = SpecifierSet(requires_python)
except InvalidSpecifier:
continue
else:
if not specifierset.contains(py_version):
continue
all_candidates.append(c)
return all_candidates
def get_specset(marker_list):
# type: (List) -> Optional[SpecifierSet]
specset = set()
_last_str = "and"
for marker_parts in marker_list:
if isinstance(marker_parts, str):
_last_str = marker_parts # noqa
else:
specset.update(_get_specifiers_from_markers(marker_parts))
specifiers = SpecifierSet()
specifiers._specs = frozenset(specset)
return specifiers
def normalize_specifier_set(specs):
# type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]]
"""Given a specifier set, a string, or an iterable, normalize the
specifiers.
.. note:: This function exists largely to deal with ``pyzmq`` which handles
the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than
the correct form of ``3.7.*``. This workaround can likely go away if
we ever introduce enforcement for metadata standards on PyPI.
:param Union[str, SpecifierSet] specs: Supplied specifiers to normalize
:return: A new set of specifiers or specifierset
:rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`]
"""
if not specs:
return None
if isinstance(specs, set):
return specs
# when we aren't dealing with a string at all, we can normalize this as usual
elif not isinstance(specs, str):
return {_format_pyspec(spec) for spec in specs}
spec_list = []
for spec in specs.split(","):
spec = spec.strip()
if spec.endswith(".*"):
spec = spec[:-2]
spec = spec.rstrip("*")
spec_list.append(spec)
return normalize_specifier_set(SpecifierSet(",".join(spec_list)))
def parse(self):
"""
Parse a Pipfile.lock (as seen in pipenv)
:return:
"""
try:
data = json.loads(self.obj.content, object_pairs_hook=OrderedDict)
if data:
for package_type in ['default', 'develop']:
if package_type in data:
for name, meta in data[package_type].items():
# skip VCS dependencies
if 'version' not in meta:
continue
specs = meta['version']
hashes = meta['hashes']
self.obj.dependencies.append(
Dependency(
name=name,
specs=SpecifierSet(specs),
dependency_type=filetypes.pipfile_lock,
hashes=hashes,
line=''.join([name, specs]),
section=package_type))
except ValueError:
pass
def fix_requires_python_marker(requires_python):
from pipenv.vendor.packaging.requirements import Requirement as PackagingRequirement
marker_str = ""
if any(
requires_python.startswith(op)
for op in Specifier._operators.keys()):
spec_dict = defaultdict(set)
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
marker_key = Variable("python_version")
for spec in specifierset:
operator, val = spec._spec
cleaned_val = Value(val).serialize().replace('"', "")
spec_dict[Op(operator).serialize()].add(cleaned_val)
marker_str = " and ".join([
"{0}{1}'{2}'".format(marker_key.serialize(), op, ",".join(vals))
for op, vals in spec_dict.items()
])
marker_to_add = PackagingRequirement(
"fakepkg; {0}".format(marker_str)).marker
return marker_to_add
def validate_specifiers(instance, attr_, value):
if value == "":
return True
try:
SpecifierSet(value)
except (InvalidMarker, InvalidSpecifier):
raise ValueError("Invalid Specifiers {0}".format(value))
def create_specifierset(spec=None):
# type: (Optional[str]) -> SpecifierSet
if isinstance(spec, SpecifierSet):
return spec
elif isinstance(spec, (set, list, tuple)):
spec = " and ".join(spec)
if spec is None:
spec = ""
return SpecifierSet(spec)
def clean_requires_python(candidates):
"""Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
all_candidates = []
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
for c in candidates:
if getattr(c, "requires_python", None):
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if len(c.requires_python) == 1 and c.requires_python in ("2", "3"):
c.requires_python = '>={0},<{1!s}'.format(c.requires_python, int(c.requires_python) + 1)
try:
specifierset = SpecifierSet(c.requires_python)
except InvalidSpecifier:
continue
else:
if not specifierset.contains(py_version):
continue
all_candidates.append(c)
return all_candidates
def check(packages, key, db_mirror, cached, ignore_ids, proxy):
key = key if key else os.environ.get("SAFETY_API_KEY", False)
db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy)
db_full = None
vulnerable_packages = frozenset(db.keys())
vulnerable = []
for pkg in packages:
# Ignore recursive files not resolved
if isinstance(pkg, RequirementFile):
continue
# normalize the package name, the safety-db is converting underscores to dashes and uses
# lowercase
name = pkg.key.replace("_", "-").lower()
if name in vulnerable_packages:
# we have a candidate here, build the spec set
for specifier in db[name]:
spec_set = SpecifierSet(specifiers=specifier)
if spec_set.contains(pkg.version):
if not db_full:
db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy)
for data in get_vulnerabilities(pkg=name, spec=specifier, db=db_full):
vuln_id = data.get("id").replace("pyup.io-", "")
cve_id = data.get("cve")
if cve_id:
cve_id = cve_id.split(",")[0].strip()
if vuln_id and vuln_id not in ignore_ids:
cve_meta = db_full.get("$meta", {}).get("cve", {}).get(cve_id, {})
vulnerable.append(
Vulnerability(
name=name,
spec=specifier,
version=pkg.version,
advisory=data.get("advisory"),
vuln_id=vuln_id,
cvssv2=cve_meta.get("cvssv2", None),
cvssv3=cve_meta.get("cvssv3", None),
)
)
return vulnerable
def _get_specs(specset):
if specset is None:
return
if is_instance(specset, Specifier):
new_specset = SpecifierSet()
specs = set()
specs.add(specset)
new_specset._specs = frozenset(specs)
specset = new_specset
if isinstance(specset, str):
specset = SpecifierSet(specset)
result = []
for spec in set(specset):
version = spec.version
op = spec.operator
if op in ("in", "not in"):
versions = version.split(",")
op = "==" if op == "in" else "!="
for ver in versions:
result.append((op, _tuplize_version(ver.strip())))
else:
result.append((spec.operator, _tuplize_version(spec.version)))
return sorted(result, key=operator.itemgetter(1))
def from_info(cls, info):
# type: ("PackageInfo") -> "Dependency"
marker_str = ""
specset_str, py_version_str = "", ""
if info.requires_python:
# XXX: Some markers are improperly formatted -- we already handle most cases
# XXX: but learned about new broken formats, such as
# XXX: python_version in "2.6 2.7 3.2 3.3" (note the lack of commas)
# XXX: as a marker on a dependency of a library called 'pickleshare'
# XXX: Some packages also have invalid markers with stray characters,
# XXX: such as 'algoliasearch'
try:
marker = marker_from_specifier(info.requires_python)
except Exception:
marker_str = ""
else:
if not marker or not marker._markers:
marker_str = ""
else:
marker_str = "{0!s}".format(marker)
req_str = "{0}=={1}".format(info.name, info.version)
if marker_str:
req_str = "{0}; {1}".format(req_str, marker_str)
req = PackagingRequirement(req_str)
requires_python_str = (
info.requires_python if info.requires_python is not None else ""
)
if req.specifier:
specset_str = str(req.specifier)
if requires_python_str:
py_version_str = requires_python_str
return cls(
name=info.name,
specifier=req.specifier,
extras=tuple(sorted(set(req.extras)))
if req.extras is not None
else req.extras,
requirement=req,
from_extras=None,
python_version=SpecifierSet(requires_python_str),
markers=None,
parent=None,
specset_str=specset_str,
python_version_str=py_version_str,
marker_str=marker_str,
)
def parse(self):
"""
Parse a Pipfile (as seen in pipenv)
:return:
"""
try:
data = toml.loads(self.obj.content, _dict=OrderedDict)
if data:
for package_type in ['packages', 'dev-packages']:
if package_type in data:
for name, specs in data[package_type].items():
# skip on VCS dependencies
if not isinstance(specs, str):
continue
if specs == '*':
specs = ''
self.obj.dependencies.append(
Dependency(name=name,
specs=SpecifierSet(specs),
dependency_type=filetypes.pipfile,
line=''.join([name, specs]),
section=package_type))
except (toml.TomlDecodeError, IndexError) as e:
pass
def parse_marker_dict(marker_dict):
op = marker_dict["op"]
lhs = marker_dict["lhs"]
rhs = marker_dict["rhs"]
# This is where the spec sets for each side land if we have an "or" operator
side_spec_list = []
side_markers_list = []
finalized_marker = ""
# And if we hit the end of the parse tree we use this format string to make a marker
format_string = "{lhs} {op} {rhs}"
specset = SpecifierSet()
specs = set()
# Essentially we will iterate over each side of the parsed marker if either one is
# A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset
# Union the "and" specs, intersect the "or"s to find the most appropriate range
if any(issubclass(type(side), Mapping) for side in (lhs, rhs)):
for side in (lhs, rhs):
side_specs = set()
side_markers = set()
if issubclass(type(side), Mapping):
merged_side_specs, merged_side_markers = parse_marker_dict(side)
side_specs.update(merged_side_specs)
side_markers.update(merged_side_markers)
else:
marker = _ensure_marker(side)
marker_parts = getattr(marker, "_markers", [])
if marker_parts[0][0].value == "python_version":
side_specs |= set(get_specset(marker_parts))
else:
side_markers.add(str(marker))
side_spec_list.append(side_specs)
side_markers_list.append(side_markers)
if op == "and":
# When we are "and"-ing things together, it probably makes the most sense
# to reduce them here into a single PySpec instance
specs = reduce(lambda x, y: set(x) | set(y), side_spec_list)
markers = reduce(lambda x, y: set(x) | set(y), side_markers_list)
if not specs and not markers:
return specset, finalized_marker
if markers and isinstance(markers, (tuple, list, Set)):
finalized_marker = Marker(" and ".join([m for m in markers if m]))
elif markers:
finalized_marker = str(markers)
specset._specs = frozenset(specs)
return specset, finalized_marker
# Actually when we "or" things as well we can also just turn them into a reduced
# set using this logic now
sides = reduce(lambda x, y: set(x) & set(y), side_spec_list)
finalized_marker = " or ".join(
[normalize_marker_str(m) for m in side_markers_list]
)
specset._specs = frozenset(sorted(sides))
return specset, finalized_marker
else:
# At the tip of the tree we are dealing with strings all around and they just need
# to be smashed together
specs = set()
if lhs == "python_version":
format_string = "{lhs}{op}{rhs}"
marker = Marker(format_string.format(**marker_dict))
marker_parts = getattr(marker, "_markers", [])
_set = get_specset(marker_parts)
if _set:
specs |= set(_set)
specset._specs = frozenset(specs)
return specset, finalized_marker
