def get(self, ecosystem):
query = request.args.get('q')
eco = Ecosystem.by_name(rdb.session, ecosystem)
fetcher = CucosReleasesFetcher(eco, rdb.session)
now = datetime.datetime.now()
# Instantiate two different solvers, one using a custom fetcher to fetch
# matching releases from Bayesian DB and the other one fetching from
# upstream repositories.
# The data from these two solvers then provide information as to:
# 1) Which packages in the range we have already analysed and have information
# about
# 2) Other packages from upstream repositories which match the version specification
cucos_solver, solver = get_ecosystem_solver(eco, with_fetcher=fetcher),\
get_ecosystem_solver(eco)
ours = cucos_solver.solve([query], all_versions=True)
upstream = solver.solve([query], all_versions=True)
ours_nums = set() if not ours else set(next(iter(ours.values())))
upstreams_nums = set() if not upstream else set(
next(iter(upstream.values())))
return {
'query': query,
'detail': {
'analysed': ours,
'upstream': upstream,
'difference': list(upstreams_nums - ours_nums)
},
'resolved_at': str(now)
}
def _handle_external_deps(self, ecosystem, deps):
"""Resolve external dependency specifications"""
if not ecosystem or not deps:
return []
solver = get_ecosystem_solver(ecosystem)
versions = solver.solve(deps)
return [{"package": k, "version": v} for k, v in versions.items()]
def _get_versions_to_scan(self, package_name, version_string,
only_already_scanned):
""" Compute versions that should be scanned based on version string
:param package_name: name of the package which versions should be resolved
:param version_string: version string for the package
:param only_already_scanned: if True analyse only packages that were already analysed
:return: list of versions that should be analysed
"""
solver = get_ecosystem_solver(self.storage.get_ecosystem('npm'))
try:
resolved = solver.solve(
["{} {}".format(package_name, version_string)],
all_versions=True)
except:
self.log.exception("Failed to resolve versions for package '%s'",
package_name)
return []
resolved_versions = resolved.get(package_name, [])
if only_already_scanned:
result = []
for version in resolved_versions:
if self.storage.get_analysis_count('npm', package_name,
version) > 0:
result.append(version)
return result
else:
return resolved_versions
def _resolve_dependency(self, ecosystem, dep):
ret = {
'ecosystem': ecosystem.name,
'declaration': dep,
'resolved_at': json_serial(datetime.datetime.now())
}
# first, if this is a Github dependency, return it right away (we don't resolve these yet)
if ' ' in dep:
name, spec = dep.split(' ', 1)
if gh_dep.match(spec):
ret['name'] = name
ret['version'] = 'https://github.com/' + spec
else:
if gh_dep.match(dep):
ret['name'] = 'https://github.com/' + dep
ret['version'] = None
if 'name' in ret:
return ret
# second, figure out what is the latest upstream version matching the spec and return it
solver = get_ecosystem_solver(ecosystem)
pkgspec = solver.solve([dep])
if not pkgspec:
raise TaskError("invalid dependency: {}".format(dep))
package, version = pkgspec.popitem()
if not version:
raise TaskError("bad version resolved for {}".format(dep))
ret['name'] = package
ret['version'] = version
return ret
def _npm_scan(self, arguments):
"""
Query Snyk vulndb stored on S3
"""
s3 = StoragePool.get_connected_storage('S3Snyk')
try:
self.log.debug('Retrieving Snyk vulndb from S3')
vulndb = s3.retrieve_vulndb()
except:
self.log.error('Failed to obtain Snyk vulndb database')
return {'summary': ['Failed to obtain Snyk vulndb database'],
'status': 'error',
'details': []}
entries = []
solver = get_ecosystem_solver(self.storage.get_ecosystem('npm'))
for entry in vulndb.get('npm', {}).get(arguments['name'], []):
vulnerable_versions = entry['semver']['vulnerable']
affected_versions = solver.solve(["{} {}".format(arguments['name'],
vulnerable_versions)],
all_versions=True)
if arguments['version'] in affected_versions.get(arguments['name'], []):
entries.append(self._filter_vulndb_fields(entry))
return {'summary': [e['id'] for e in entries if e],
'status': 'success',
'details': entries}
def test_rubygems(self, rubygems):
solver = get_ecosystem_solver(rubygems)
deps = [
"Hoe ~>3.14", "rexicaL >=1.0.5", "raKe-compiler-dock ~>0.4.2",
"rake-comPiler ~>0.9.2"
]
out = solver.solve(deps)
assert len(out) == len(deps)
def test_npm_solver(self, npm, semver_string, expected):
solver = get_ecosystem_solver(npm)
name = 'test_name'
# mock fetched releases to have predictable results
flexmock(NpmReleasesFetcher,
fetch_releases=(name, self.SERVE_STATIC_VER))
solver_result = solver.solve([name + ' ' + semver_string],
all_versions=True)
# {'name': ['1.0.0', '1.0.1']}
assert set(solver_result.get(name, [])) == set(expected)
def test_cucos_fetcher(self, rdb, npm):
# create initial dataset
package = Package(ecosystem=npm, name='cucos')
rdb.add(package)
rdb.commit()
versions = {
'0.5.0', '0.5.1', '0.6.0', '0.6.4', '0.7.0', '0.8.0', '0.9.0',
'1.0.0', '1.0.5'
}
for v in versions:
version = Version(package=package, identifier=v)
rdb.add(version)
rdb.commit()
analysis = Analysis(version=version)
# Fetcher only selects finished analyses
analysis.finished_at = datetime.datetime.now()
rdb.add(analysis)
rdb.commit()
f = CucosReleasesFetcher(npm, rdb)
r = f.fetch_releases('cucos')[1]
# make sure we fetched the same stuff we inserted
assert set(r) == versions
# first should be the latest
assert r.pop() == '1.0.5'
# try different dependency specs
s = get_ecosystem_solver(npm, f)
assert s.solve(['cucos ^0.5.0'])['cucos'] == '0.5.1'
assert s.solve(['cucos 0.x.x'])['cucos'] == '0.9.0'
assert s.solve(['cucos >1.0.0'])['cucos'] == '1.0.5'
assert s.solve(['cucos ~>0.6.0'])['cucos'] == '0.6.4'
# check that with `all_versions` we return all the relevant ones
assert set(s.solve(['cucos >=0.6.0'], all_versions=True)['cucos']) == \
(versions - {'0.5.0', '0.5.1'})
def test_pypi(self, pypi):
solver = get_ecosystem_solver(pypi)
deps = ["pymongo>=3.0,<3.2.2", "celery>3.1.11", "six==1.10.0"]
out = solver.solve(deps)
assert len(out) == len(deps)
def test_maven_solver(self, maven, dependencies, expected):
solver = get_ecosystem_solver(maven)
solver_result = solver.solve(dependencies)
assert len(solver_result) == len(dependencies)
for name, version in solver_result.items():
assert expected.get(name, '') == version