def _get_normalized_packages(): six = Package(name='six', version='1.2') pip = Package(name='pip', version='20.1') foo = Package(**{ 'name': 'flask', 'version': '0.12', 'dependencies': [ { 'name': 'six', 'version': '1.2' }, { 'name': 'six', 'version': '1.2' }, { 'name': 'flask', 'version': '0.12' }] }) bar = Package(**{ 'name': 'bar', 'version': '0.12', 'dependencies': [Package(**six.dict()), Package(**pip.dict())] }) return NormalizedPackages([foo, bar], 'pypi')
def test_normalized_packages_basic_transitive(): """Test NormalizedPackages with transitives dependency.""" flask = Package(name='flask', version='0.12') six = Package(name='six', version='1.2.3') foo = Package(**{ 'name': flask.name, 'version': flask.version, 'dependencies': [{ 'name': six.name, 'version': six.version }] }) assert foo is not None normalized = NormalizedPackages([foo], 'pypi') assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 assert flask in normalized.direct_dependencies # transtive should have an entry assert len(normalized.transitive_dependencies) == 1 assert six in normalized.transitive_dependencies # all must be 2 assert len(normalized.all_dependencies) == 2 assert flask in normalized.all_dependencies assert six in normalized.all_dependencies # dependency graph assert len(normalized.dependency_graph) == 1 assert len(normalized.dependency_graph[flask]) == 1 assert flask in normalized.dependency_graph assert six in normalized.dependency_graph[flask] assert flask not in normalized.dependency_graph[flask]
def test_normalized_packages_basic_direct(): """Test NormalizedPackages with basic dependency.""" pkg = Package(name='flask', version='0.12') foo = Package(**{ 'name': pkg.name, 'version': pkg.version }) assert foo is not None normalized = NormalizedPackages([foo], 'pypi') assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 assert pkg in normalized.direct_dependencies # transtives must be empty assert len(normalized.transitive_dependencies) == 0 # all must be 1 assert normalized.all_dependencies is not None assert len(normalized.all_dependencies) == 1 assert pkg in normalized.all_dependencies # dependency_graph assert len(normalized.dependency_graph) == 1 assert pkg in normalized.dependency_graph assert len(normalized.dependency_graph[foo]) == 0
def __init__(self, packages: List[Package], ecosystem: Ecosystem): """Create NormalizedPackages by removing all duplicates from packages.""" self._packages = packages self._ecosystem = ecosystem self._dependency_graph: Dict[Package, Set[Package]] = defaultdict(set) for package in packages: # clone without dependencies field if ecosystem == "golang": package.name = package.name.split("@")[0] _, package.version = GolangDependencyTreeGenerator.clean_version( package.version) package_clone = Package(name=package.name, version=package.version) self._dependency_graph[package_clone] = self._dependency_graph[ package_clone] or set() for trans_package in package.dependencies or []: trans_clone = Package(name=trans_package.name, version=trans_package.version) self._dependency_graph[package].add(trans_clone) # unfold set of Package into flat set of Package self._transtives: Set[Package] = { d for dep in self._dependency_graph.values() for d in dep } self._directs = frozenset(self._dependency_graph.keys()) self._all = self._directs.union(self._transtives)
def test_pkg_non_equvality(): """Test not equvality.""" pkg_0 = Package(name='flask', version='0.12') pkg_1 = Package(name='flask', version='0.13') assert pkg_0 != pkg_1 pkg_0 = Package(name='flask', version='0.12') pkg_1 = Package(name='django', version='0.12') assert pkg_0 != pkg_1
def test_recommendation_response_with_empty_graph(_mock_insight_response, _mock_gremlin, client): """Test with empty graph response.""" response = client.post( "/api/v2/recommender?persist=false", data=json.dumps( RecommenderRequest( registration_status="FREETIER", external_request_id="foo", ecosystem="npm", manifest_file_path="/foo.bar", packages=[Package(name="request", version="2.88.2")], ).dict()), content_type="application/json", ) assert response.status_code == 200 response = get_json_from_response(response) assert response["result"] == { "companion": [], "external_request_id": "foo", "manifest_file_path": "/foo.bar", "manifest_name": None, "recommendation_status": "success", "registration_status": "FREETIER", "usage_outliers": [], "uuid": None, }
def _get_pkg_from_graph_version_node( version_node) -> Tuple[Ecosystem, Package]: """Create Package instance from version_node.""" name = version_node.get("pname", [""])[0] version = version_node.get("version", [""])[0] ecosystem = version_node.get("pecosystem", [""])[0] return ecosystem, Package(name=name, version=version)
def _get_normalized_packages(): six = Package(name='six', version='1.2') pip = Package(name='pip', version='20.1') flask = Package( **{ 'name': 'flask', 'version': '0.12', 'dependencies': [{ 'name': 'flask-mock', 'version': '0.0.13' }] }) bar = Package(**{ 'name': 'bar', 'version': '0.12', 'dependencies': [flask, six, pip] }) return NormalizedPackages([flask, bar], 'pypi')
def test_normalized_packages_golang(): """Test Normalised Golang Packages.""" pkg = Package(name='github.com/pingcap/tidb/[email protected]/api', version='v2.0.0-rc.3') assert pkg is not None normalized = NormalizedPackages([pkg], Ecosystem.golang) assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 # transitives must be empty assert len(normalized.transitive_dependencies) == 0
def test_normalized_packages_with_duplicates(): """Test NormalizedPackages with duplicates.""" flask = Package(name='flask', version='0.12') six = Package(name='six', version='1.2') normalized = _get_normalized_packages() assert normalized.ecosystem == 'pypi' assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 2 assert flask in normalized.direct_dependencies assert six not in normalized.direct_dependencies # transtive should have an entry assert len(normalized.transitive_dependencies) == 3 assert six in normalized.transitive_dependencies assert flask in normalized.transitive_dependencies assert len(normalized.all_dependencies) == 4 assert flask in normalized.all_dependencies assert six in normalized.all_dependencies
def _get_golang_package_details(self, pkg_node) -> Tuple[Package, PackageDetails]: """Get Pseudo Golang Package Details.""" pkg_name = pkg_node.get('name', [None])[0] ecosystem = pkg_node.get('ecosystem', [''])[0] pkg = Package(name=pkg_name, version=self._normalized_packages.version_map[pkg_name]) latest_version = pkg_node.get('latest_version', [''])[0] public_vulns, private_vulns = _get_vulnerabilities( self.filtered_vul.get(pkg_name, [])) recommended_latest_version = pkg_node.get("latest_non_cve_version", [""])[0] pkg_details = PackageDataWithVulnerabilities( **pkg.dict(), ecosystem=ecosystem, latest_version=latest_version, github={}, licenses=[], url=get_snyk_package_link(ecosystem, pkg_name), private_vulnerabilities=private_vulns, public_vulnerabilities=public_vulns, recommended_version=recommended_latest_version) return pkg, pkg_details
def clean_and_get_pkgs(packages) -> Tuple[List[Package], List[str]]: """Clean and get golang packages.""" all_packages: List[Package] = [] all_modules: List[str] = [] gh = GithubUtils() for direct in packages: pkg_name, pkg_mod = get_golang_metadata(direct) _, package_version = GolangDependencyTreeGenerator.clean_version( direct.version) pkg = Package(name=pkg_name, version=package_version, dependencies=[]) if gh.is_pseudo_version(package_version): all_modules.append(pkg_mod) for trans_pkg in direct.dependencies or []: trans_name, trans_mod = get_golang_metadata(trans_pkg) _, trans_version = GolangDependencyTreeGenerator.clean_version( trans_pkg.version) trans = Package(name=trans_name, version=trans_version) pkg.dependencies.append(trans) if gh.is_pseudo_version(trans_version): all_modules.append(trans_mod) all_packages.append(pkg) return all_packages, all_modules
def test_recommendation_response_with_insight_error(_mock_insight_response, _mock_gremlin, client): """Test golang ecosystem, it should return empty result.""" response = client.post( "/api/v2/recommender?persist=false", data=json.dumps( RecommenderRequest( registration_status="", external_request_id="", ecosystem="npm", manifest_file_path="", packages=[Package(name="request", version="2.88.2")], ).dict()), content_type="application/json", ) assert response.status_code == 500
def _get_normalized_packages(): flask = Package(name='flask', version='0.12') six = Package(name='six', version='3.1.1') flask_details = PackageDataWithVulnerabilities(**flask.dict(), latest_version='1.1.2', ecosystem='pypi', licenses=['ABC']) six_details = PackageDataWithVulnerabilities(**six.dict(), latest_version='3.5.0', ecosystem='pypi', licenses=['XYZ', 'ABC']) return [flask_details, six_details]
def test_pkg_hashing(): """Test hashing functionality of Package.""" pkg_0 = Package(name='flask', version='0.12') pkg_1 = Package(name='flask', version='0.12') pkg_2 = Package(name='flask', version='0.13') pkg_3 = Package(name='django', version='0.13') pkg_4 = Package(name='flask', version='0.12') set_of_pkgs = set([pkg_0, pkg_1, pkg_2, pkg_3, pkg_4]) assert len(set_of_pkgs) == 3 assert pkg_0 in set_of_pkgs assert pkg_1 in set_of_pkgs assert pkg_2 in set_of_pkgs assert pkg_3 in set_of_pkgs assert pkg_4 in set_of_pkgs assert Package(name='bar', version='0.0') not in set_of_pkgs
def test_pkg_equvality(): """Test package equvality.""" pkg_0 = Package(name='flask', version='0.12') pkg_1 = Package(name='flask', version='0.12') assert pkg_0 == pkg_1
"""Tests for the sa module.""" import copy import json from unittest import mock, TestCase from src.v2 import stack_aggregator as sa from src.v2.stack_aggregator import StackAggregator from src.v2.models import (Package, VulnerabilityFields, StackAggregatorResult, StackAggregatorRequest) from src.v2.normalized_packages import NormalizedPackages _DJANGO = Package(name='django', version='1.2.1') _FLASK = Package(name='flask', version='0.12') _SIX = Package(name='six', version='3.2.1') _FOO_UNKNOWN = Package(name='foo_unknown', version='0.0.0') def _request_body(): return { "registration_status": "REGISTERED", "uuid": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "manifest_name": "requirements.txt", "manifest_file_path": "/foo/bar", "external_request_id": "test_id", "ecosystem":
def test_recommendation_response_with_2_packages_from_graph( _mock_insight_response, _mock_gremlin, client): """Test for happy scenario.""" response = client.post( "/api/v2/recommender?persist=false", data=json.dumps( RecommenderRequest( registration_status="FREETIER", external_request_id="foo", ecosystem="npm", manifest_file_path="/foo.bar", packages=[Package(name="request", version="2.88.2")], ).dict()), content_type="application/json", ) assert response.status_code == 200 response = get_json_from_response(response) assert response["result"] == { "companion": [ { "cooccurrence_count": 0, "cooccurrence_probability": 52.50463667254613, "dependencies": None, "ecosystem": "npm", "github": { "contributors": "238", "dependent_projects": "48988", "dependent_repos": "893994", "forks_count": "8571", "issues": { "month": { "closed": 18, "opened": 18 }, "year": { "closed": -1, "opened": -1 }, }, "latest_release_duration": "2018-10-27 03:12:11", "open_issues_count": "155", "pull_requests": { "month": { "closed": 0, "opened": 3 }, "year": { "closed": 61, "opened": 75 }, }, "size": "N/A", "stargazers_count": "51280", "total_releases": "291", "used_by": [ { "name": "Automattic/mongoose", "stars": "14372" }, { "name": "ReactTraining/react-router", "stars": "27232" }, { "name": "angular/angular.js", "stars": "57749" }, { "name": "angular/angular", "stars": "31547" }, { "name": "facebook/jest", "stars": "14465" }, { "name": "facebook/react", "stars": "84669" }, { "name": "postcss/autoprefixer", "stars": "14331" }, { "name": "reactjs/redux", "stars": "36922" }, { "name": "socketio/socket.io", "stars": "38209" }, { "name": "webpack/webpack", "stars": "35429" }, ], "watchers": "1799", }, "latest_version": "4.17.1", "licenses": [], "name": "express", "topic_list": ["accepts"], "url": "https://snyk.io/vuln/npm:express", "version": "4.17.1", }, { "cooccurrence_count": 0, "cooccurrence_probability": 51.04134789351682, "dependencies": None, "ecosystem": "npm", "github": { "contributors": "30", "dependent_projects": "6413", "dependent_repos": "35152", "forks_count": "1129", "issues": { "month": { "closed": -1, "opened": -1 }, "year": { "closed": -1, "opened": -1 }, }, "latest_release_duration": "2018-06-12 16:31:26", "open_issues_count": "236", "pull_requests": { "month": { "closed": -1, "opened": -1 }, "year": { "closed": -1, "opened": -1 }, }, "size": "N/A", "stargazers_count": "12020", "total_releases": "73", "used_by": [ { "name": "amir20/phantomjs-node", "stars": "2884" }, { "name": "angular/angular.js", "stars": "56712" }, { "name": "angular/angular", "stars": "26987" }, { "name": "angular/material2", "stars": "10337" }, { "name": "apidoc/apidoc", "stars": "4685" }, { "name": "cyclejs/cyclejs", "stars": "7180" }, { "name": "flatiron/prompt", "stars": "1142" }, { "name": "foreverjs/forever", "stars": "9868" }, { "name": "ionic-team/ionic-native", "stars": "1384" }, { "name": "ionic-team/ionic", "stars": "30889" }, ], "watchers": "224", }, "latest_version": "3.2.1", "licenses": [], "name": "winston", "topic_list": ["async"], "url": "https://snyk.io/vuln/npm:winston", "version": "3.2.1", }, ], "external_request_id": "foo", "manifest_file_path": "/foo.bar", "manifest_name": None, "recommendation_status": "success", "registration_status": "FREETIER", "usage_outliers": [], "uuid": None, }
def test_pkg_basic(): """Test basic package properties.""" pkg = Package(name='flask', version='0.12') assert pkg.name == 'flask' assert pkg.version == '0.12'
def test_normalized_packages_dependency_graph(): """Test NormalizedPackages dependency_graph.""" flask = Package(name='flask', version='0.12') six = Package(name='six', version='1.2') pip = Package(name='pip', version='20.1') foo = Package(**{ 'name': 'flask', 'version': '0.12', }) bar = Package(**{ 'name': 'bar', 'version': '0.12', 'dependencies': [Package(**six.dict()), Package(**pip.dict())] }) normalized = _get_normalized_packages() assert foo in normalized.dependency_graph assert bar in normalized.dependency_graph assert flask in normalized.dependency_graph assert six not in normalized.dependency_graph assert six in normalized.dependency_graph[foo] assert foo in normalized.dependency_graph[foo] assert flask in normalized.dependency_graph[foo] assert pip not in normalized.dependency_graph[foo] assert pip in normalized.dependency_graph[bar] assert six in normalized.dependency_graph[bar]