def test_golang_solver(self, go, dependencies, expected): solver = get_ecosystem_solver(go) solver_result = solver.solve(dependencies) assert len(solver_result) == len(dependencies) for name, version in solver_result.items(): assert expected.get(name, '') == version, '"{}" "{}" "{}"'.format( name, version, expected)
def _query_ossindex(self, arguments): """Query OSS Index REST API.""" entries = {} solver = get_ecosystem_solver(self.storage.get_ecosystem( arguments['ecosystem']), with_parser=OSSIndexDependencyParser()) for package in self._query_ossindex_package(arguments['ecosystem'], arguments['name']): for vulnerability in package.get('vulnerabilities', []): for version_string in vulnerability.get('versions', []): try: affected_versions = solver.solve([ "{} {}".format(arguments['name'], version_string) ], all_versions=True) except Exception: self.log.exception("Failed to resolve %r for %s:%s", version_string, arguments['ecosystem'], arguments['name']) continue if arguments['version'] in affected_versions.get( arguments['name'], []): entry = self._filter_ossindex_fields(vulnerability) if entry.get('id'): entries[entry['id']] = entry return { 'summary': list(entries.keys()), 'status': 'success', 'details': list(entries.values()) }
def test_maven_solver(self, maven, dependencies, expected): """Test MavenSolver.""" solver = get_ecosystem_solver(maven) solver_result = solver.solve(dependencies) assert len(solver_result) == len(dependencies) for name, version in solver_result.items(): assert expected.get(name, '') == version
def test_rubygems_solver(self, rubygems): solver = get_ecosystem_solver(rubygems) deps = ['hoe <3.4.0', 'rake-compiler ~>0.9.2'] out = solver.solve(deps) assert out == {'hoe': '3.3.1', 'rake-compiler': '0.9.9'}
def test_npm_solver(self, npm, semver_string, expected): solver = get_ecosystem_solver(npm) name = 'test_name' # mock fetched releases to have predictable results flexmock(NpmReleasesFetcher, fetch_releases=(name, self.SERVE_STATIC_VER)) solver_result = solver.solve([name + ' ' + semver_string], all_versions=True) # {'name': ['1.0.0', '1.0.1']} assert set(solver_result.get(name, [])) == set(expected)
def components_to_scan(self, previous_sync_timestamp, only_already_scanned): """Get EPV that were recently updated in OSS Index, so they can contain new vulnerabilities. Get components (e:p:v) that were recently (since previous_sync_timestamp) updated in OSS Index, which means that they can contain new vulnerabilities. :param previous_sync_timestamp: timestamp of previous check :param only_already_scanned: include already scanned components only :return: generator of e:p:v """ # TODO: reduce cyclomatic complexity to_scan = [] for ecosystem in ['nuget']: ecosystem_solver = get_ecosystem_solver(self.storage.get_ecosystem(ecosystem), with_parser=OSSIndexDependencyParser()) self.log.debug("Retrieving new %s vulnerabilities from OSS Index", ecosystem) ossindex_updated_packages = CVEcheckerTask.\ query_ossindex_vulnerability_fromtill(ecosystem=ecosystem, from_time=previous_sync_timestamp) for ossindex_updated_package in ossindex_updated_packages: if ecosystem == 'maven': package_name = "{g}:{n}".format(g=ossindex_updated_package['group'], n=ossindex_updated_package['name']) else: package_name = ossindex_updated_package['name'] package_affected_versions = set() for vulnerability in ossindex_updated_package.get('vulnerabilities', []): for version_string in vulnerability.get('versions', []): try: resolved_versions = ecosystem_solver.\ solve(["{} {}".format(package_name, version_string)], all_versions=True) except Exception: self.log.exception("Failed to resolve %r for %s:%s", version_string, ecosystem, package_name) continue resolved_versions = resolved_versions.get(package_name, []) if only_already_scanned: already_scanned_versions =\ [ver for ver in resolved_versions if self.storage.get_analysis_count(ecosystem, package_name, ver) > 0] package_affected_versions.update(already_scanned_versions) else: package_affected_versions.update(resolved_versions) for version in package_affected_versions: to_scan.append({ 'ecosystem': ecosystem, 'name': package_name, 'version': version }) msg = "Components to be {prefix}scanned for vulnerabilities: {components}".\ format(prefix="re-" if only_already_scanned else "", components=to_scan) self.log.info(msg) return to_scan
def _handle_external_deps(ecosystem, deps): """Resolve external dependency specifications.""" if not ecosystem or not deps: return [] solver = get_ecosystem_solver(ecosystem) try: versions = solver.solve(deps) except Exception as exc: raise FatalTaskError("Dependencies could not be resolved: '{}'" .format(deps)) from exc return [{"package": k, "version": v} for k, v in versions.items()]
def test_nuget_solver(self, nuget): solver = get_ecosystem_solver(nuget) deps = ['jQuery [1.4.4, 1.6)', 'NUnit 3.2.1', 'NETStandard.Library [1.6.0, )'] out = solver.solve(deps) # nuget resolves to lowest version by default, see # https://docs.microsoft.com/en-us/nuget/release-notes/nuget-2.8#-dependencyversion-switch assert out == {'jQuery': '1.4.4', 'NUnit': '3.2.1', 'NETStandard.Library': '1.6.0'}
def _handle_external_deps(ecosystem, deps): """Resolve external dependency specifications.""" if not ecosystem or not deps: return [] solver = get_ecosystem_solver(ecosystem) try: versions = solver.solve(deps) except Exception: logger.error('Dependencies could not be resolved: "%s"', deps) raise return [{"package": k, "version": v} for k, v in versions.items()]
def test_pypi_solver(self, pypi): solver = get_ecosystem_solver(pypi) deps = ['django == 1.9.10', 'pymongo >=3.0, <3.2.2', 'six~=1.7.1', 'requests===2.16.2', 'click==0.*'] out = solver.solve(deps) assert out == {'django': '1.9.10', 'pymongo': '3.2.1', 'six': '1.7.3', 'requests': '2.16.2', 'click': '0.7'}
def test_pypi_solver(self, pypi): """Test PypiSolver.""" solver = get_ecosystem_solver(pypi) deps = ['django == 1.9.10', 'pymongo >=3.0, <3.2.2', 'six~=1.7.1', 'coverage~=3.5.1b1.dev', 'pyasn1>=0.2.2,~=0.2.2', 'requests===2.16.2', 'click==0.*'] out = solver.solve(deps) assert out == {'django': '1.9.10', 'pymongo': '3.2.1', 'six': '1.7.3', 'coverage': '3.5.3', 'pyasn1': '0.2.3', 'requests': '2.16.2', 'click': '0.7'}
def _resolve_dependency(ecosystem, dep): ret = { 'ecosystem': ecosystem.name, 'declaration': dep, 'resolved_at': json_serial(datetime.datetime.utcnow()) } # first, if this is a Github dependency, return it right away (we don't resolve these yet) if ' ' in dep: # we have both package name and version (version can be an URL) name, spec = dep.split(' ', 1) if gh_dep.match(spec): ret['name'] = name ret['version'] = 'https://github.com/' + spec elif urllib.parse.urlparse(spec).scheme is not '': ret['name'] = name ret['version'] = spec else: if gh_dep.match(dep): ret['name'] = 'https://github.com/' + dep ret['version'] = None elif urllib.parse.urlparse(dep).scheme is not '': ret['name'] = dep ret['version'] = None if 'name' in ret: return ret # second, figure out what is the latest upstream version matching the spec and return it solver = get_ecosystem_solver(ecosystem) pkgspec = solver.solve([dep]) if not pkgspec: raise TaskError("invalid dependency: {}".format(dep)) package, version = pkgspec.popitem() if not version: raise TaskError("could not resolve {}".format(dep)) ret['name'] = package ret['version'] = version return ret
def test_f8a_fetcher(self, rdb, npm): """Test F8aReleasesFetcher.""" # create initial dataset package = Package(ecosystem=npm, name='f8a') rdb.add(package) rdb.commit() versions = { '0.5.0', '0.5.1', '0.6.0', '0.6.4', '0.7.0', '0.8.0', '0.9.0', '1.0.0', '1.0.5' } for v in versions: version = Version(package=package, identifier=v) rdb.add(version) rdb.commit() analysis = Analysis(version=version) # Fetcher only selects finished analyses analysis.finished_at = datetime.datetime.utcnow() rdb.add(analysis) rdb.commit() f = F8aReleasesFetcher(npm, rdb) r = f.fetch_releases('f8a')[1] # make sure we fetched the same stuff we inserted assert set(r) == versions # first should be the latest assert r.pop() == '1.0.5' # try different dependency specs s = get_ecosystem_solver(npm, with_fetcher=f) assert s.solve(['f8a ^0.5.0'])['f8a'] == '0.5.1' assert s.solve(['f8a 0.x.x'])['f8a'] == '0.9.0' assert s.solve(['f8a >1.0.0'])['f8a'] == '1.0.5' assert s.solve(['f8a ~>0.6.0'])['f8a'] == '0.6.4' # check that with `all_versions` we return all the relevant ones assert set(s.solve(['f8a >=0.6.0'], all_versions=True)['f8a']) == \ (versions - {'0.5.0', '0.5.1'})