def get_latest_version_within_specs(specs, versions, prereleases=None): spec_set = SpecifierSet(",".join(["".join([x, y]) for x, y in specs])) candidates = [] for version in versions: if spec_set.contains(version, prereleases=prereleases): candidates.append(version) candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True) if len(candidates) > 0: return candidates[0] return None
def get_latest_version_within_specs(specs, versions, prereleases=None): # build up a spec set and convert compatible specs to pinned ones spec_set = SpecifierSet( ",".join(["".join([x.replace("~=", "=="), y]) for x, y in specs]) ) candidates = [] for version in versions: if spec_set.contains(version, prereleases=prereleases): candidates.append(version) candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True) if len(candidates) > 0: return candidates[0] return None
def get_latest_version_within_specs(specs, versions, prereleases=None): # build up a spec set and convert compatible specs to pinned ones spec_set = SpecifierSet(",".join( ["".join([x.replace("~=", "=="), y]) for x, y in specs])) candidates = [] for version in versions: if spec_set.contains(version, prereleases=prereleases): candidates.append(version) candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True) if len(candidates) > 0: return candidates[0] return None
def get_latest_version_within_specs(specs, versions, prereleases=None): spec_set = SpecifierSet(",".join(["".join([x, y]) for x, y in specs])) candidates = [] # print("specs are", spec_set) for version in versions: if spec_set.contains(version, prereleases=prereleases): candidates.append(version) # else: # print(spec_set, "does not contain", version) candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True) if len(candidates) > 0: # print("candidates are", candidates) return candidates[0] return None
def get_renovate_config(project_path: Path) -> Dict[str, SpecifierSet]: if not SpecifierSet: return {} renovate_json = project_path.joinpath('renovate.json') if not renovate_json.is_file(): return {} with renovate_json.open('r', encoding='utf-8') as fh: data = json.load(fh) try: python_config: Dict[str, dict] = data['python'] python_pkg_rules: List[Dict[str, Union[ List[str], str]]] = python_config['packageRules'] except KeyError: return {} constraints: Dict[str, str] = {} for rule in python_pkg_rules: try: names: List[str] = rule['packageNames'] allowed_versions: str = rule['allowedVersions'] except KeyError: continue constraints.update( {name.lower(): SpecifierSet(allowed_versions) for name in names}) return constraints
def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() repo_name, image_tag = image.split('/')[-1].split(':') ignore_ids_list = _get_safety_ignore_list(image) sep = " -i " ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}" container_name = f"{repo_name}-{image_tag}-safety" docker_exec_cmd = f"docker exec -i {container_name}" test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety") # Add null entrypoint to ensure command exits immediately run( f"docker run -id " f"--name {container_name} " f"--mount type=bind,src=$(pwd)/container_tests,target=/test " f"--entrypoint='/bin/bash' " f"{image}", hide=True) try: run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) json_str_safety_result = safety_check.run_safety_check_on_container( docker_exec_cmd) safety_result = json.loads(json_str_safety_result) for package, affected_versions, curr_version, _, vulnerability_id in safety_result: # Get the latest version of the package with vulnerability command_success, latest_version = _get_latest_package_version( docker_exec_cmd, package) if not command_success: continue # If the latest version of the package is also affected, ignore this vulnerability if Version(latest_version) in SpecifierSet(affected_versions): # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. # Comparing two versions as a string has some edge cases which require us to write more code. # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and # gives an object that can be easily compared against a Version object. # https://packaging.pypa.io/en/latest/specifiers/ ignore_str += f" -i {vulnerability_id}" assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ f"Safety test failed for {image}" finally: run(f"docker rm -f {container_name}", hide=True)
def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. """ repo_name, image_tag = image.split('/')[-1].split(':') ignore_ids_list = _get_safety_ignore_list(image) ignore_str = "" if not ignore_ids_list else " ".join(ignore_ids_list) container_name = f"{repo_name}-{image_tag}-safety" docker_exec_cmd = f"docker exec -i {container_name}" test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety") # Add null entrypoint to ensure command exits immediately run(f"docker run -id " f"--name {container_name} " f"--mount type=bind,src=$(pwd)/container_tests,target=/test " f"--entrypoint='/bin/bash' " f"{image}", hide=True) try: run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) run_out = run(f"{docker_exec_cmd} safety check --json ", warn=True, hide=True) json_str_safety_result = run_out.stdout safety_result = json.loads(json_str_safety_result) for package, affected_versions, curr_version, _, vulnerability_id in safety_result: run_out = run(f"{docker_exec_cmd} yolk -M {package} -f version ", warn=True, hide=True) if run_out.return_code != 0: continue latest_version = run_out.stdout if Version(latest_version) in SpecifierSet(affected_versions): # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. # Comparing two versions as a string has some edge cases which require us to write more code. # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and # gives an object that can be easily compared against a Version object. # https://packaging.pypa.io/en/latest/specifiers/ ignore_str += f" {vulnerability_id}" run(f"{docker_exec_cmd} chmod +x {test_file_path}", hide=True) output = run(f"{docker_exec_cmd} {test_file_path} {ignore_str} ", warn=True) LOGGER.info(f"{test_file_path} log for {image}\n{output.stdout}") assert output.return_code == 0, f"Safety test failed for {image}\n{output.stdout}" finally: run(f"docker rm -f {container_name}", hide=True)
def __init__(self, line, build=None): self.build = build super(Requirement, self).__init__(line) self.name = self.name.lower().replace('_', '-') self.specifier = SpecifierSet(','.join(f"{op}{ver}" for op, ver in self.specs))
def __init__(self, line): super(Requirement, self).__init__(line) self.name = self.name.lower().replace('_', '-') self.specs = list(self.norm_specs(self.specs)) self.specifier = SpecifierSet(','.join(f"{op}{ver}" for op, ver in self.specs))
def package_data(name): """ Creates detailed package data, see django.json :param name: package name """ with open('index.json') as f: item = json.loads(f.read()).get(name, {}) data = { "count": sum([n for _, n in item.items()]), "specs": { "unpinned": 0, "range": 0, "pinned": 0, "compatible": 0, "unknown": 0 }, "releases": { "unknown": 0 }, "major_releases": {}, "security": { "secure": 0, "insecure": 0, "unknown": 0 } } # calculate specs for spec, count in item.items(): if spec == "": data['specs']['unpinned'] += count elif spec.startswith("=="): data['specs']['pinned'] += count elif spec.startswith("~="): data['specs']['compatible'] += count else: is_range_spec = False for range_spec in ('<', ">", "!="): if spec.startswith(range_spec): data['specs']['range'] += count is_range_spec = True break if not is_range_spec: data['specs']['unknown'] += count # releases import requests r = requests.get(f"https://pypi.python.org/pypi/{name}/json") releases = sorted(r.json()["releases"].keys(), key=lambda v: parse_version(v), reverse=True) for spec, count in item.items(): if not spec.startswith("=="): data['releases']['unknown'] += count continue spec_set = SpecifierSet(spec) candidates = [] for release in releases: if spec_set.contains(release, prereleases=True): candidates.append(release) candidates = sorted(candidates, key=lambda v: parse_version(v), reverse=True) if len(candidates) > 0: key = str(candidates[0]) if key not in data['releases']: data['releases'][key] = 0 data['releases'][key] += count else: data['releases']['unknown'] += count from collections import OrderedDict data['releases'] = OrderedDict(sorted(data['releases'].items(), key=lambda v: parse_version(v[0]), reverse=True)) # major releases for release, count in data['releases'].items(): major_release = ".".join(release.split(".")[:2]) if major_release not in data["major_releases"]: data["major_releases"][major_release] = 0 data["major_releases"][major_release] += count data['major_releases'] = OrderedDict(sorted(data['major_releases'].items(), key=lambda v: parse_version(v[0]), reverse=True)) # security for release, count in data['releases'].items(): if release == 'unknown': data['security']['unknown'] = count continue vulns = safety_check( packages=safety_read_requirements(StringIO(f"{name}=={release}")), key="", db_mirror="", cached=True, ignore_ids=[] ) if len(vulns) > 0: data['security']['insecure'] += count else: data['security']['secure'] += count return data