def test_golang_solver(self, go, dependencies, expected):
solver = get_ecosystem_solver(go)
solver_result = solver.solve(dependencies)
assert len(solver_result) == len(dependencies)
for name, version in solver_result.items():
assert expected.get(name, '') == version, '"{}" "{}" "{}"'.format(
name, version, expected)
def _query_ossindex(self, arguments):
"""Query OSS Index REST API."""
entries = {}
solver = get_ecosystem_solver(self.storage.get_ecosystem(
arguments['ecosystem']),
with_parser=OSSIndexDependencyParser())
for package in self._query_ossindex_package(arguments['ecosystem'],
arguments['name']):
for vulnerability in package.get('vulnerabilities', []):
for version_string in vulnerability.get('versions', []):
try:
affected_versions = solver.solve([
"{} {}".format(arguments['name'], version_string)
],
all_versions=True)
except Exception:
self.log.exception("Failed to resolve %r for %s:%s",
version_string,
arguments['ecosystem'],
arguments['name'])
continue
if arguments['version'] in affected_versions.get(
arguments['name'], []):
entry = self._filter_ossindex_fields(vulnerability)
if entry.get('id'):
entries[entry['id']] = entry
return {
'summary': list(entries.keys()),
'status': 'success',
'details': list(entries.values())
}
def test_maven_solver(self, maven, dependencies, expected):
"""Test MavenSolver."""
solver = get_ecosystem_solver(maven)
solver_result = solver.solve(dependencies)
assert len(solver_result) == len(dependencies)
for name, version in solver_result.items():
assert expected.get(name, '') == version
def test_rubygems_solver(self, rubygems):
solver = get_ecosystem_solver(rubygems)
deps = ['hoe <3.4.0',
'rake-compiler ~>0.9.2']
out = solver.solve(deps)
assert out == {'hoe': '3.3.1',
'rake-compiler': '0.9.9'}
def test_npm_solver(self, npm, semver_string, expected):
solver = get_ecosystem_solver(npm)
name = 'test_name'
# mock fetched releases to have predictable results
flexmock(NpmReleasesFetcher, fetch_releases=(name, self.SERVE_STATIC_VER))
solver_result = solver.solve([name + ' ' + semver_string], all_versions=True)
# {'name': ['1.0.0', '1.0.1']}
assert set(solver_result.get(name, [])) == set(expected)
def components_to_scan(self, previous_sync_timestamp, only_already_scanned):
"""Get EPV that were recently updated in OSS Index, so they can contain new vulnerabilities.
Get components (e:p:v) that were recently (since previous_sync_timestamp) updated
in OSS Index, which means that they can contain new vulnerabilities.
:param previous_sync_timestamp: timestamp of previous check
:param only_already_scanned: include already scanned components only
:return: generator of e:p:v
"""
# TODO: reduce cyclomatic complexity
to_scan = []
for ecosystem in ['nuget']:
ecosystem_solver = get_ecosystem_solver(self.storage.get_ecosystem(ecosystem),
with_parser=OSSIndexDependencyParser())
self.log.debug("Retrieving new %s vulnerabilities from OSS Index", ecosystem)
ossindex_updated_packages = CVEcheckerTask.\
query_ossindex_vulnerability_fromtill(ecosystem=ecosystem,
from_time=previous_sync_timestamp)
for ossindex_updated_package in ossindex_updated_packages:
if ecosystem == 'maven':
package_name = "{g}:{n}".format(g=ossindex_updated_package['group'],
n=ossindex_updated_package['name'])
else:
package_name = ossindex_updated_package['name']
package_affected_versions = set()
for vulnerability in ossindex_updated_package.get('vulnerabilities', []):
for version_string in vulnerability.get('versions', []):
try:
resolved_versions = ecosystem_solver.\
solve(["{} {}".format(package_name, version_string)],
all_versions=True)
except Exception:
self.log.exception("Failed to resolve %r for %s:%s", version_string,
ecosystem, package_name)
continue
resolved_versions = resolved_versions.get(package_name, [])
if only_already_scanned:
already_scanned_versions =\
[ver for ver in resolved_versions if
self.storage.get_analysis_count(ecosystem, package_name, ver) > 0]
package_affected_versions.update(already_scanned_versions)
else:
package_affected_versions.update(resolved_versions)
for version in package_affected_versions:
to_scan.append({
'ecosystem': ecosystem,
'name': package_name,
'version': version
})
msg = "Components to be {prefix}scanned for vulnerabilities: {components}".\
format(prefix="re-" if only_already_scanned else "",
components=to_scan)
self.log.info(msg)
return to_scan
def _handle_external_deps(ecosystem, deps):
"""Resolve external dependency specifications."""
if not ecosystem or not deps:
return []
solver = get_ecosystem_solver(ecosystem)
try:
versions = solver.solve(deps)
except Exception as exc:
raise FatalTaskError("Dependencies could not be resolved: '{}'" .format(deps)) from exc
return [{"package": k, "version": v} for k, v in versions.items()]
def test_nuget_solver(self, nuget):
solver = get_ecosystem_solver(nuget)
deps = ['jQuery [1.4.4, 1.6)',
'NUnit 3.2.1',
'NETStandard.Library [1.6.0, )']
out = solver.solve(deps)
# nuget resolves to lowest version by default, see
# https://docs.microsoft.com/en-us/nuget/release-notes/nuget-2.8#-dependencyversion-switch
assert out == {'jQuery': '1.4.4',
'NUnit': '3.2.1',
'NETStandard.Library': '1.6.0'}
def _handle_external_deps(ecosystem, deps):
"""Resolve external dependency specifications."""
if not ecosystem or not deps:
return []
solver = get_ecosystem_solver(ecosystem)
try:
versions = solver.solve(deps)
except Exception:
logger.error('Dependencies could not be resolved: "%s"', deps)
raise
return [{"package": k, "version": v} for k, v in versions.items()]
def test_pypi_solver(self, pypi):
solver = get_ecosystem_solver(pypi)
deps = ['django == 1.9.10',
'pymongo >=3.0, <3.2.2',
'six~=1.7.1',
'requests===2.16.2',
'click==0.*']
out = solver.solve(deps)
assert out == {'django': '1.9.10',
'pymongo': '3.2.1',
'six': '1.7.3',
'requests': '2.16.2',
'click': '0.7'}
def test_pypi_solver(self, pypi):
"""Test PypiSolver."""
solver = get_ecosystem_solver(pypi)
deps = ['django == 1.9.10',
'pymongo >=3.0, <3.2.2',
'six~=1.7.1',
'coverage~=3.5.1b1.dev',
'pyasn1>=0.2.2,~=0.2.2',
'requests===2.16.2',
'click==0.*']
out = solver.solve(deps)
assert out == {'django': '1.9.10',
'pymongo': '3.2.1',
'six': '1.7.3',
'coverage': '3.5.3',
'pyasn1': '0.2.3',
'requests': '2.16.2',
'click': '0.7'}
def _resolve_dependency(ecosystem, dep):
ret = {
'ecosystem': ecosystem.name,
'declaration': dep,
'resolved_at': json_serial(datetime.datetime.utcnow())
}
# first, if this is a Github dependency, return it right away (we don't resolve these yet)
if ' ' in dep:
# we have both package name and version (version can be an URL)
name, spec = dep.split(' ', 1)
if gh_dep.match(spec):
ret['name'] = name
ret['version'] = 'https://github.com/' + spec
elif urllib.parse.urlparse(spec).scheme is not '':
ret['name'] = name
ret['version'] = spec
else:
if gh_dep.match(dep):
ret['name'] = 'https://github.com/' + dep
ret['version'] = None
elif urllib.parse.urlparse(dep).scheme is not '':
ret['name'] = dep
ret['version'] = None
if 'name' in ret:
return ret
# second, figure out what is the latest upstream version matching the spec and return it
solver = get_ecosystem_solver(ecosystem)
pkgspec = solver.solve([dep])
if not pkgspec:
raise TaskError("invalid dependency: {}".format(dep))
package, version = pkgspec.popitem()
if not version:
raise TaskError("could not resolve {}".format(dep))
ret['name'] = package
ret['version'] = version
return ret
def test_f8a_fetcher(self, rdb, npm):
"""Test F8aReleasesFetcher."""
# create initial dataset
package = Package(ecosystem=npm, name='f8a')
rdb.add(package)
rdb.commit()
versions = {
'0.5.0', '0.5.1', '0.6.0', '0.6.4', '0.7.0', '0.8.0', '0.9.0',
'1.0.0', '1.0.5'
}
for v in versions:
version = Version(package=package, identifier=v)
rdb.add(version)
rdb.commit()
analysis = Analysis(version=version)
# Fetcher only selects finished analyses
analysis.finished_at = datetime.datetime.utcnow()
rdb.add(analysis)
rdb.commit()
f = F8aReleasesFetcher(npm, rdb)
r = f.fetch_releases('f8a')[1]
# make sure we fetched the same stuff we inserted
assert set(r) == versions
# first should be the latest
assert r.pop() == '1.0.5'
# try different dependency specs
s = get_ecosystem_solver(npm, with_fetcher=f)
assert s.solve(['f8a ^0.5.0'])['f8a'] == '0.5.1'
assert s.solve(['f8a 0.x.x'])['f8a'] == '0.9.0'
assert s.solve(['f8a >1.0.0'])['f8a'] == '1.0.5'
assert s.solve(['f8a ~>0.6.0'])['f8a'] == '0.6.4'
# check that with `all_versions` we return all the relevant ones
assert set(s.solve(['f8a >=0.6.0'], all_versions=True)['f8a']) == \
(versions - {'0.5.0', '0.5.1'})
