def _get_normalized_packages():
six = Package(name='six', version='1.2')
pip = Package(name='pip', version='20.1')
foo = Package(**{
'name': 'flask',
'version': '0.12',
'dependencies': [
{
'name': 'six',
'version': '1.2'
},
{
'name': 'six',
'version': '1.2'
},
{
'name': 'flask',
'version': '0.12'
}]
})
bar = Package(**{
'name': 'bar',
'version': '0.12',
'dependencies': [Package(**six.dict()), Package(**pip.dict())]
})
return NormalizedPackages([foo, bar], 'pypi')
def test_normalized_packages_basic_transitive():
"""Test NormalizedPackages with transitives dependency."""
flask = Package(name='flask', version='0.12')
six = Package(name='six', version='1.2.3')
foo = Package(**{
'name': flask.name,
'version': flask.version,
'dependencies': [{
'name': six.name,
'version': six.version
}]
})
assert foo is not None
normalized = NormalizedPackages([foo], 'pypi')
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
assert flask in normalized.direct_dependencies
# transtive should have an entry
assert len(normalized.transitive_dependencies) == 1
assert six in normalized.transitive_dependencies
# all must be 2
assert len(normalized.all_dependencies) == 2
assert flask in normalized.all_dependencies
assert six in normalized.all_dependencies
# dependency graph
assert len(normalized.dependency_graph) == 1
assert len(normalized.dependency_graph[flask]) == 1
assert flask in normalized.dependency_graph
assert six in normalized.dependency_graph[flask]
assert flask not in normalized.dependency_graph[flask]
def test_normalized_packages_basic_direct():
"""Test NormalizedPackages with basic dependency."""
pkg = Package(name='flask', version='0.12')
foo = Package(**{
'name': pkg.name,
'version': pkg.version
})
assert foo is not None
normalized = NormalizedPackages([foo], 'pypi')
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
assert pkg in normalized.direct_dependencies
# transtives must be empty
assert len(normalized.transitive_dependencies) == 0
# all must be 1
assert normalized.all_dependencies is not None
assert len(normalized.all_dependencies) == 1
assert pkg in normalized.all_dependencies
# dependency_graph
assert len(normalized.dependency_graph) == 1
assert pkg in normalized.dependency_graph
assert len(normalized.dependency_graph[foo]) == 0
def __init__(self, packages: List[Package], ecosystem: Ecosystem):
"""Create NormalizedPackages by removing all duplicates from packages."""
self._packages = packages
self._ecosystem = ecosystem
self._dependency_graph: Dict[Package, Set[Package]] = defaultdict(set)
for package in packages:
# clone without dependencies field
if ecosystem == "golang":
package.name = package.name.split("@")[0]
_, package.version = GolangDependencyTreeGenerator.clean_version(
package.version)
package_clone = Package(name=package.name, version=package.version)
self._dependency_graph[package_clone] = self._dependency_graph[
package_clone] or set()
for trans_package in package.dependencies or []:
trans_clone = Package(name=trans_package.name,
version=trans_package.version)
self._dependency_graph[package].add(trans_clone)
# unfold set of Package into flat set of Package
self._transtives: Set[Package] = {
d
for dep in self._dependency_graph.values() for d in dep
}
self._directs = frozenset(self._dependency_graph.keys())
self._all = self._directs.union(self._transtives)
def test_pkg_non_equvality():
"""Test not equvality."""
pkg_0 = Package(name='flask', version='0.12')
pkg_1 = Package(name='flask', version='0.13')
assert pkg_0 != pkg_1
pkg_0 = Package(name='flask', version='0.12')
pkg_1 = Package(name='django', version='0.12')
assert pkg_0 != pkg_1
def test_recommendation_response_with_empty_graph(_mock_insight_response,
_mock_gremlin, client):
"""Test with empty graph response."""
response = client.post(
"/api/v2/recommender?persist=false",
data=json.dumps(
RecommenderRequest(
registration_status="FREETIER",
external_request_id="foo",
ecosystem="npm",
manifest_file_path="/foo.bar",
packages=[Package(name="request", version="2.88.2")],
).dict()),
content_type="application/json",
)
assert response.status_code == 200
response = get_json_from_response(response)
assert response["result"] == {
"companion": [],
"external_request_id": "foo",
"manifest_file_path": "/foo.bar",
"manifest_name": None,
"recommendation_status": "success",
"registration_status": "FREETIER",
"usage_outliers": [],
"uuid": None,
}
def _get_pkg_from_graph_version_node(
version_node) -> Tuple[Ecosystem, Package]:
"""Create Package instance from version_node."""
name = version_node.get("pname", [""])[0]
version = version_node.get("version", [""])[0]
ecosystem = version_node.get("pecosystem", [""])[0]
return ecosystem, Package(name=name, version=version)
def _get_normalized_packages():
six = Package(name='six', version='1.2')
pip = Package(name='pip', version='20.1')
flask = Package(
**{
'name': 'flask',
'version': '0.12',
'dependencies': [{
'name': 'flask-mock',
'version': '0.0.13'
}]
})
bar = Package(**{
'name': 'bar',
'version': '0.12',
'dependencies': [flask, six, pip]
})
return NormalizedPackages([flask, bar], 'pypi')
def test_normalized_packages_golang():
"""Test Normalised Golang Packages."""
pkg = Package(name='github.com/pingcap/tidb/[email protected]/api',
version='v2.0.0-rc.3')
assert pkg is not None
normalized = NormalizedPackages([pkg], Ecosystem.golang)
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
# transitives must be empty
assert len(normalized.transitive_dependencies) == 0
def test_normalized_packages_with_duplicates():
"""Test NormalizedPackages with duplicates."""
flask = Package(name='flask', version='0.12')
six = Package(name='six', version='1.2')
normalized = _get_normalized_packages()
assert normalized.ecosystem == 'pypi'
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 2
assert flask in normalized.direct_dependencies
assert six not in normalized.direct_dependencies
# transtive should have an entry
assert len(normalized.transitive_dependencies) == 3
assert six in normalized.transitive_dependencies
assert flask in normalized.transitive_dependencies
assert len(normalized.all_dependencies) == 4
assert flask in normalized.all_dependencies
assert six in normalized.all_dependencies
def _get_golang_package_details(self, pkg_node) -> Tuple[Package, PackageDetails]:
"""Get Pseudo Golang Package Details."""
pkg_name = pkg_node.get('name', [None])[0]
ecosystem = pkg_node.get('ecosystem', [''])[0]
pkg = Package(name=pkg_name, version=self._normalized_packages.version_map[pkg_name])
latest_version = pkg_node.get('latest_version', [''])[0]
public_vulns, private_vulns = _get_vulnerabilities(
self.filtered_vul.get(pkg_name, []))
recommended_latest_version = pkg_node.get("latest_non_cve_version", [""])[0]
pkg_details = PackageDataWithVulnerabilities(
**pkg.dict(),
ecosystem=ecosystem,
latest_version=latest_version,
github={},
licenses=[],
url=get_snyk_package_link(ecosystem, pkg_name),
private_vulnerabilities=private_vulns,
public_vulnerabilities=public_vulns,
recommended_version=recommended_latest_version)
return pkg, pkg_details
def clean_and_get_pkgs(packages) -> Tuple[List[Package], List[str]]:
"""Clean and get golang packages."""
all_packages: List[Package] = []
all_modules: List[str] = []
gh = GithubUtils()
for direct in packages:
pkg_name, pkg_mod = get_golang_metadata(direct)
_, package_version = GolangDependencyTreeGenerator.clean_version(
direct.version)
pkg = Package(name=pkg_name, version=package_version, dependencies=[])
if gh.is_pseudo_version(package_version):
all_modules.append(pkg_mod)
for trans_pkg in direct.dependencies or []:
trans_name, trans_mod = get_golang_metadata(trans_pkg)
_, trans_version = GolangDependencyTreeGenerator.clean_version(
trans_pkg.version)
trans = Package(name=trans_name, version=trans_version)
pkg.dependencies.append(trans)
if gh.is_pseudo_version(trans_version):
all_modules.append(trans_mod)
all_packages.append(pkg)
return all_packages, all_modules
def test_recommendation_response_with_insight_error(_mock_insight_response,
_mock_gremlin, client):
"""Test golang ecosystem, it should return empty result."""
response = client.post(
"/api/v2/recommender?persist=false",
data=json.dumps(
RecommenderRequest(
registration_status="",
external_request_id="",
ecosystem="npm",
manifest_file_path="",
packages=[Package(name="request", version="2.88.2")],
).dict()),
content_type="application/json",
)
assert response.status_code == 500
def _get_normalized_packages():
flask = Package(name='flask', version='0.12')
six = Package(name='six', version='3.1.1')
flask_details = PackageDataWithVulnerabilities(**flask.dict(),
latest_version='1.1.2',
ecosystem='pypi',
licenses=['ABC'])
six_details = PackageDataWithVulnerabilities(**six.dict(),
latest_version='3.5.0',
ecosystem='pypi',
licenses=['XYZ', 'ABC'])
return [flask_details, six_details]
def test_pkg_hashing():
"""Test hashing functionality of Package."""
pkg_0 = Package(name='flask', version='0.12')
pkg_1 = Package(name='flask', version='0.12')
pkg_2 = Package(name='flask', version='0.13')
pkg_3 = Package(name='django', version='0.13')
pkg_4 = Package(name='flask', version='0.12')
set_of_pkgs = set([pkg_0, pkg_1, pkg_2, pkg_3, pkg_4])
assert len(set_of_pkgs) == 3
assert pkg_0 in set_of_pkgs
assert pkg_1 in set_of_pkgs
assert pkg_2 in set_of_pkgs
assert pkg_3 in set_of_pkgs
assert pkg_4 in set_of_pkgs
assert Package(name='bar', version='0.0') not in set_of_pkgs
def test_pkg_equvality():
"""Test package equvality."""
pkg_0 = Package(name='flask', version='0.12')
pkg_1 = Package(name='flask', version='0.12')
assert pkg_0 == pkg_1
"""Tests for the sa module."""
import copy
import json
from unittest import mock, TestCase
from src.v2 import stack_aggregator as sa
from src.v2.stack_aggregator import StackAggregator
from src.v2.models import (Package, VulnerabilityFields, StackAggregatorResult,
StackAggregatorRequest)
from src.v2.normalized_packages import NormalizedPackages
_DJANGO = Package(name='django', version='1.2.1')
_FLASK = Package(name='flask', version='0.12')
_SIX = Package(name='six', version='3.2.1')
_FOO_UNKNOWN = Package(name='foo_unknown', version='0.0.0')
def _request_body():
return {
"registration_status":
"REGISTERED",
"uuid":
"3fa85f64-5717-4562-b3fc-2c963f66afa6",
"manifest_name":
"requirements.txt",
"manifest_file_path":
"/foo/bar",
"external_request_id":
"test_id",
"ecosystem":
def test_recommendation_response_with_2_packages_from_graph(
_mock_insight_response, _mock_gremlin, client):
"""Test for happy scenario."""
response = client.post(
"/api/v2/recommender?persist=false",
data=json.dumps(
RecommenderRequest(
registration_status="FREETIER",
external_request_id="foo",
ecosystem="npm",
manifest_file_path="/foo.bar",
packages=[Package(name="request", version="2.88.2")],
).dict()),
content_type="application/json",
)
assert response.status_code == 200
response = get_json_from_response(response)
assert response["result"] == {
"companion": [
{
"cooccurrence_count": 0,
"cooccurrence_probability": 52.50463667254613,
"dependencies": None,
"ecosystem": "npm",
"github": {
"contributors":
"238",
"dependent_projects":
"48988",
"dependent_repos":
"893994",
"forks_count":
"8571",
"issues": {
"month": {
"closed": 18,
"opened": 18
},
"year": {
"closed": -1,
"opened": -1
},
},
"latest_release_duration":
"2018-10-27 03:12:11",
"open_issues_count":
"155",
"pull_requests": {
"month": {
"closed": 0,
"opened": 3
},
"year": {
"closed": 61,
"opened": 75
},
},
"size":
"N/A",
"stargazers_count":
"51280",
"total_releases":
"291",
"used_by": [
{
"name": "Automattic/mongoose",
"stars": "14372"
},
{
"name": "ReactTraining/react-router",
"stars": "27232"
},
{
"name": "angular/angular.js",
"stars": "57749"
},
{
"name": "angular/angular",
"stars": "31547"
},
{
"name": "facebook/jest",
"stars": "14465"
},
{
"name": "facebook/react",
"stars": "84669"
},
{
"name": "postcss/autoprefixer",
"stars": "14331"
},
{
"name": "reactjs/redux",
"stars": "36922"
},
{
"name": "socketio/socket.io",
"stars": "38209"
},
{
"name": "webpack/webpack",
"stars": "35429"
},
],
"watchers":
"1799",
},
"latest_version": "4.17.1",
"licenses": [],
"name": "express",
"topic_list": ["accepts"],
"url": "https://snyk.io/vuln/npm:express",
"version": "4.17.1",
},
{
"cooccurrence_count": 0,
"cooccurrence_probability": 51.04134789351682,
"dependencies": None,
"ecosystem": "npm",
"github": {
"contributors":
"30",
"dependent_projects":
"6413",
"dependent_repos":
"35152",
"forks_count":
"1129",
"issues": {
"month": {
"closed": -1,
"opened": -1
},
"year": {
"closed": -1,
"opened": -1
},
},
"latest_release_duration":
"2018-06-12 16:31:26",
"open_issues_count":
"236",
"pull_requests": {
"month": {
"closed": -1,
"opened": -1
},
"year": {
"closed": -1,
"opened": -1
},
},
"size":
"N/A",
"stargazers_count":
"12020",
"total_releases":
"73",
"used_by": [
{
"name": "amir20/phantomjs-node",
"stars": "2884"
},
{
"name": "angular/angular.js",
"stars": "56712"
},
{
"name": "angular/angular",
"stars": "26987"
},
{
"name": "angular/material2",
"stars": "10337"
},
{
"name": "apidoc/apidoc",
"stars": "4685"
},
{
"name": "cyclejs/cyclejs",
"stars": "7180"
},
{
"name": "flatiron/prompt",
"stars": "1142"
},
{
"name": "foreverjs/forever",
"stars": "9868"
},
{
"name": "ionic-team/ionic-native",
"stars": "1384"
},
{
"name": "ionic-team/ionic",
"stars": "30889"
},
],
"watchers":
"224",
},
"latest_version": "3.2.1",
"licenses": [],
"name": "winston",
"topic_list": ["async"],
"url": "https://snyk.io/vuln/npm:winston",
"version": "3.2.1",
},
],
"external_request_id":
"foo",
"manifest_file_path":
"/foo.bar",
"manifest_name":
None,
"recommendation_status":
"success",
"registration_status":
"FREETIER",
"usage_outliers": [],
"uuid":
None,
}
def test_pkg_basic():
"""Test basic package properties."""
pkg = Package(name='flask', version='0.12')
assert pkg.name == 'flask'
assert pkg.version == '0.12'
def test_normalized_packages_dependency_graph():
"""Test NormalizedPackages dependency_graph."""
flask = Package(name='flask', version='0.12')
six = Package(name='six', version='1.2')
pip = Package(name='pip', version='20.1')
foo = Package(**{
'name': 'flask',
'version': '0.12',
})
bar = Package(**{
'name': 'bar',
'version': '0.12',
'dependencies': [Package(**six.dict()), Package(**pip.dict())]
})
normalized = _get_normalized_packages()
assert foo in normalized.dependency_graph
assert bar in normalized.dependency_graph
assert flask in normalized.dependency_graph
assert six not in normalized.dependency_graph
assert six in normalized.dependency_graph[foo]
assert foo in normalized.dependency_graph[foo]
assert flask in normalized.dependency_graph[foo]
assert pip not in normalized.dependency_graph[foo]
assert pip in normalized.dependency_graph[bar]
assert six in normalized.dependency_graph[bar]
