def get_latest_version_within_specs(specs, versions, prereleases=None):
spec_set = SpecifierSet(",".join(["".join([x, y]) for x, y in specs]))
candidates = []
for version in versions:
if spec_set.contains(version, prereleases=prereleases):
candidates.append(version)
candidates = sorted(candidates,
key=lambda v: parse_version(v),
reverse=True)
if len(candidates) > 0:
return candidates[0]
return None
def get_latest_version_within_specs(specs, versions, prereleases=None):
# build up a spec set and convert compatible specs to pinned ones
spec_set = SpecifierSet(
",".join(["".join([x.replace("~=", "=="), y]) for x, y in specs])
)
candidates = []
for version in versions:
if spec_set.contains(version, prereleases=prereleases):
candidates.append(version)
candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True)
if len(candidates) > 0:
return candidates[0]
return None
def get_latest_version_within_specs(specs, versions, prereleases=None):
# build up a spec set and convert compatible specs to pinned ones
spec_set = SpecifierSet(",".join(
["".join([x.replace("~=", "=="), y]) for x, y in specs]))
candidates = []
for version in versions:
if spec_set.contains(version, prereleases=prereleases):
candidates.append(version)
candidates = sorted(candidates,
key=lambda v: parse_version(v),
reverse=True)
if len(candidates) > 0:
return candidates[0]
return None
def get_latest_version_within_specs(specs, versions, prereleases=None):
spec_set = SpecifierSet(",".join(["".join([x, y]) for x, y in specs]))
candidates = []
# print("specs are", spec_set)
for version in versions:
if spec_set.contains(version, prereleases=prereleases):
candidates.append(version)
# else:
# print(spec_set, "does not contain", version)
candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True)
if len(candidates) > 0:
# print("candidates are", candidates)
return candidates[0]
return None
def get_renovate_config(project_path: Path) -> Dict[str, SpecifierSet]:
if not SpecifierSet:
return {}
renovate_json = project_path.joinpath('renovate.json')
if not renovate_json.is_file():
return {}
with renovate_json.open('r', encoding='utf-8') as fh:
data = json.load(fh)
try:
python_config: Dict[str, dict] = data['python']
python_pkg_rules: List[Dict[str, Union[
List[str], str]]] = python_config['packageRules']
except KeyError:
return {}
constraints: Dict[str, str] = {}
for rule in python_pkg_rules:
try:
names: List[str] = rule['packageNames']
allowed_versions: str = rule['allowedVersions']
except KeyError:
continue
constraints.update(
{name.lower(): SpecifierSet(allowed_versions)
for name in names})
return constraints
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
from dlc.safety_check import SafetyCheck
safety_check = SafetyCheck()
repo_name, image_tag = image.split('/')[-1].split(':')
ignore_ids_list = _get_safety_ignore_list(image)
sep = " -i "
ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}"
container_name = f"{repo_name}-{image_tag}-safety"
docker_exec_cmd = f"docker exec -i {container_name}"
test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety")
# Add null entrypoint to ensure command exits immediately
run(
f"docker run -id "
f"--name {container_name} "
f"--mount type=bind,src=$(pwd)/container_tests,target=/test "
f"--entrypoint='/bin/bash' "
f"{image}",
hide=True)
try:
run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True)
json_str_safety_result = safety_check.run_safety_check_on_container(
docker_exec_cmd)
safety_result = json.loads(json_str_safety_result)
for package, affected_versions, curr_version, _, vulnerability_id in safety_result:
# Get the latest version of the package with vulnerability
command_success, latest_version = _get_latest_package_version(
docker_exec_cmd, package)
if not command_success:
continue
# If the latest version of the package is also affected, ignore this vulnerability
if Version(latest_version) in SpecifierSet(affected_versions):
# Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet.
# Comparing two versions as a string has some edge cases which require us to write more code.
# SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and
# gives an object that can be easily compared against a Version object.
# https://packaging.pypa.io/en/latest/specifiers/
ignore_str += f" -i {vulnerability_id}"
assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \
f"Safety test failed for {image}"
finally:
run(f"docker rm -f {container_name}", hide=True)
def test_safety(image):
"""
Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise
error if an issue is fixable.
"""
repo_name, image_tag = image.split('/')[-1].split(':')
ignore_ids_list = _get_safety_ignore_list(image)
ignore_str = "" if not ignore_ids_list else " ".join(ignore_ids_list)
container_name = f"{repo_name}-{image_tag}-safety"
docker_exec_cmd = f"docker exec -i {container_name}"
test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety")
# Add null entrypoint to ensure command exits immediately
run(f"docker run -id "
f"--name {container_name} "
f"--mount type=bind,src=$(pwd)/container_tests,target=/test "
f"--entrypoint='/bin/bash' "
f"{image}", hide=True)
try:
run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True)
run_out = run(f"{docker_exec_cmd} safety check --json ", warn=True, hide=True)
json_str_safety_result = run_out.stdout
safety_result = json.loads(json_str_safety_result)
for package, affected_versions, curr_version, _, vulnerability_id in safety_result:
run_out = run(f"{docker_exec_cmd} yolk -M {package} -f version ", warn=True, hide=True)
if run_out.return_code != 0:
continue
latest_version = run_out.stdout
if Version(latest_version) in SpecifierSet(affected_versions):
# Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet.
# Comparing two versions as a string has some edge cases which require us to write more code.
# SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and
# gives an object that can be easily compared against a Version object.
# https://packaging.pypa.io/en/latest/specifiers/
ignore_str += f" {vulnerability_id}"
run(f"{docker_exec_cmd} chmod +x {test_file_path}", hide=True)
output = run(f"{docker_exec_cmd} {test_file_path} {ignore_str} ", warn=True)
LOGGER.info(f"{test_file_path} log for {image}\n{output.stdout}")
assert output.return_code == 0, f"Safety test failed for {image}\n{output.stdout}"
finally:
run(f"docker rm -f {container_name}", hide=True)
def __init__(self, line, build=None):
self.build = build
super(Requirement, self).__init__(line)
self.name = self.name.lower().replace('_', '-')
self.specifier = SpecifierSet(','.join(f"{op}{ver}"
for op, ver in self.specs))
def __init__(self, line):
super(Requirement, self).__init__(line)
self.name = self.name.lower().replace('_', '-')
self.specs = list(self.norm_specs(self.specs))
self.specifier = SpecifierSet(','.join(f"{op}{ver}" for op, ver in self.specs))
def package_data(name):
"""
Creates detailed package data, see django.json
:param name: package name
"""
with open('index.json') as f:
item = json.loads(f.read()).get(name, {})
data = {
"count": sum([n for _, n in item.items()]),
"specs": {
"unpinned": 0,
"range": 0,
"pinned": 0,
"compatible": 0,
"unknown": 0
},
"releases": {
"unknown": 0
},
"major_releases": {},
"security": {
"secure": 0,
"insecure": 0,
"unknown": 0
}
}
# calculate specs
for spec, count in item.items():
if spec == "":
data['specs']['unpinned'] += count
elif spec.startswith("=="):
data['specs']['pinned'] += count
elif spec.startswith("~="):
data['specs']['compatible'] += count
else:
is_range_spec = False
for range_spec in ('<', ">", "!="):
if spec.startswith(range_spec):
data['specs']['range'] += count
is_range_spec = True
break
if not is_range_spec:
data['specs']['unknown'] += count
# releases
import requests
r = requests.get(f"https://pypi.python.org/pypi/{name}/json")
releases = sorted(r.json()["releases"].keys(), key=lambda v: parse_version(v), reverse=True)
for spec, count in item.items():
if not spec.startswith("=="):
data['releases']['unknown'] += count
continue
spec_set = SpecifierSet(spec)
candidates = []
for release in releases:
if spec_set.contains(release, prereleases=True):
candidates.append(release)
candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True)
if len(candidates) > 0:
key = str(candidates[0])
if key not in data['releases']:
data['releases'][key] = 0
data['releases'][key] += count
else:
data['releases']['unknown'] += count
from collections import OrderedDict
data['releases'] = OrderedDict(sorted(data['releases'].items(), key=lambda v: parse_version(v[0]), reverse=True))
# major releases
for release, count in data['releases'].items():
major_release = ".".join(release.split(".")[:2])
if major_release not in data["major_releases"]:
data["major_releases"][major_release] = 0
data["major_releases"][major_release] += count
data['major_releases'] = OrderedDict(sorted(data['major_releases'].items(), key=lambda v: parse_version(v[0]), reverse=True))
# security
for release, count in data['releases'].items():
if release == 'unknown':
data['security']['unknown'] = count
continue
vulns = safety_check(
packages=safety_read_requirements(StringIO(f"{name}=={release}")),
key="",
db_mirror="",
cached=True,
ignore_ids=[]
)
if len(vulns) > 0:
data['security']['insecure'] += count
else:
data['security']['secure'] += count
return data