def get_batch_ca_data(ecosystem: str, packages) -> dict:
"""Fetch package details for component analyses."""
logger.debug('Executing get_batch_ca_data')
started_at = time.time()
response = None
semver_packages = []
pseudo_version_packages = []
# Need to seperate semver and pseudo verion packages for golang
if (ecosystem == "golang"):
for p in packages:
if p['is_pseudo_version']:
pseudo_version_packages.append(p)
else:
semver_packages.append(p)
else:
semver_packages = packages
if len(semver_packages) > 0:
response = GraphAnalyses.get_batch_ca_data(ecosystem, semver_packages)
if len(pseudo_version_packages) > 0:
pseudo_response = GraphAnalyses.get_batch_ca_data_for_pseudo_version(
ecosystem, pseudo_version_packages)
# Merge both data into one.
if response:
response['result']['data'] += pseudo_response['result']['data']
else:
response = pseudo_response
elapsed_time = time.time() - started_at
logger.info("It took %s to fetch results from Gremlin.", elapsed_time)
return response if response else {}
def get_component_analyses_response(self) -> Optional[Dict]:
"""Fetch analysis for given package+version from the graph database.
Fetch analysis for given package+version from the graph database.
Main Package for Building Vendor Response.
It finally builds and returns JSON Response. This is vendor specific Function.
:returns:
- Json Response
- None: On exceptions.
"""
logger.info('Executing Vendor Specific Analyses')
try:
graph_response = GraphAnalyses.get_ca_data_from_graph(
self.ecosystem, self.package, self.version, vendor='snyk')
if not self.is_package_known(graph_response):
# Trigger Unknown Flow on Unknown Packages
logger.info(f"Package {self.package} is not known.")
return None
return ComponentAnalysisResponseBuilder(
self.ecosystem, self.package,
self.version).generate_recommendation(graph_response)
except Exception as e:
logger.error('ERROR: %s', str(e))
return None
def test_filter_vulnerable_packages_different_version(self):
"""Test vuln filtering for module/packages with different version."""
vulnerabilities = self.gremlin_vulnerabilities.get('result',
{}).get('data', [])
package_version_map = {
'github.com/crda/test': {
'v0.0.0-20160902000632-abcd4321dcba': {}
},
'github.com/crda/test/package1': {
'v0.0.0-20180902000632-abcd4321dcba': {},
'v0.0.0-20201002000632-dcbaabcd4321': {}
}
}
vuln = GraphAnalyses.filter_vulnerable_packages(
vulnerabilities, package_version_map)
self.assertIsInstance(vuln, dict)
self.assertEqual(len(vuln), 2)
self.assertIn('github.com/crda/test', vuln)
self.assertIn('github.com/crda/test/package1', vuln)
self.assertIsInstance(
vuln.get('github.com/crda/test/package1',
{}).get('v0.0.0-20180902000632-abcd4321dcba',
{}).get('cve'), list)
self.assertEqual(
len(
vuln.get('github.com/crda/test/package1',
{}).get('v0.0.0-20180902000632-abcd4321dcba',
{}).get('cve')), 1)
self.assertEqual(
vuln.get('github.com/crda/test/package1',
{}).get('v0.0.0-20181002000632-dcbaabcd4321', None), None)
def test_get_data_from_graph(self, _mockpost):
"""Test Get data from Graph. Gremlin calls."""
_mockpost().json.return_value = self.resp_json
ga = GraphAnalyses.get_ca_data_from_graph('eco', 'pkg', 'ver', 'snyk')
self.assertIsInstance(ga, dict)
self.assertIn('result', ga)
self.assertIsInstance(ga.get('result'), dict)
self.assertIn('requestId', ga)
self.assertIsInstance(ga.get('requestId'), str)
self.assertIn('status', ga)
self.assertIsInstance(ga.get('status'), dict)
def test_get_batch_ca_data(self, _mockpost):
"""Test get_batch_ca_data."""
_mockpost().json.return_value = self.gremlin_batch
ga = GraphAnalyses.get_batch_ca_data(
ecosystem='eco', packages=[{'name': 'django', 'version': '1.1'}])
self.assertIsInstance(ga, dict)
self.assertIn('result', ga)
self.assertIsInstance(ga.get('result'), dict)
self.assertIn('requestId', ga)
self.assertIsInstance(ga.get('requestId'), str)
self.assertIn('status', ga)
self.assertIsInstance(ga.get('status'), dict)
def test_get_module_package_data(self, _mockpost):
"""Test module details query from gremlin."""
_mockpost().json.return_value = self.gremlin_packages
packages = [{
'name': 'github.com/crda/test',
'version': 'v0.0.0-20160902000632-abcd4321dcba'
}]
ga = GraphAnalyses.get_package_details('eco', packages)
self.assertIsInstance(ga, dict)
self.assertIn('result', ga)
self.assertIsInstance(ga.get('result'), dict)
self.assertIn('requestId', ga)
self.assertIsInstance(ga.get('requestId'), str)
self.assertIn('status', ga)
self.assertIsInstance(ga.get('status'), dict)
def test_get_vulnerabilities_for_packages(self, _mockpost):
"""Test vulnerabilities query for packages from gremlin."""
_mockpost().json.return_value = self.gremlin_vulnerabilities
packages = [{
'name': 'github.com/crda/test/package1',
'version': 'v0.0.0-20180902000632-abcd4321dcba'
}]
ga = GraphAnalyses.get_vulnerabilities_for_packages('eco', packages)
self.assertIsInstance(ga, dict)
self.assertIn('result', ga)
self.assertIsInstance(ga.get('result'), dict)
self.assertIn('requestId', ga)
self.assertIsInstance(ga.get('requestId'), str)
self.assertIn('status', ga)
self.assertIsInstance(ga.get('status'), dict)
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Query GraphDB,
graph_response = GraphAnalyses.get_batch_ca_data(
ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW"
) != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def test_get_batch_ca_data_for_pseudo_version(self, _mockpckg, _mockvuln):
"""Test pseudo version gremlin query."""
_mockpckg.return_value = self.gremlin_packages
_mockvuln.return_value = self.gremlin_vulnerabilities
packages = [
{'name': 'github.com/crda/test', 'version': 'v0.0.0-20160902000632-abcd4321dcba',
'is_pseudo_version': True},
{'name': 'github.com/crda/test/package1',
'version': 'v0.0.0-20180902000632-abcd4321dcba', 'is_pseudo_version': True}
]
ga = GraphAnalyses.get_batch_ca_data_for_pseudo_version('eco', packages)
self.assertIsInstance(ga, dict)
self.assertIn('result', ga)
self.assertIsInstance(ga.get('result'), dict)
self.assertIn('requestId', ga)
self.assertIsInstance(ga.get('requestId'), str)
self.assertIn('status', ga)
self.assertIsInstance(ga.get('status'), dict)
def test_filter_vulnerable_packages(self):
"""Test vuln filtering for module/packages."""
vulnerabilities = self.gremlin_vulnerabilities.get('result',
{}).get('data', [])
package_version_map = {
'github.com/crda/test': 'v0.0.0-20160902000632-abcd4321dcba',
'github.com/crda/test/package1':
'v0.0.0-20180902000632-abcd4321dcba',
'github.com/crda/test/package2':
'v0.0.0-20181002000632-abcd4321dcba',
'github.com/crda/test2': 'v0.0.0-20161002000632-abcd4321dcba'
}
vuln = GraphAnalyses.filter_vulnerable_packages(
vulnerabilities, package_version_map)
self.assertIsInstance(vuln, dict)
self.assertEqual(len(vuln), 2)
self.assertIn('github.com/crda/test', vuln)
self.assertIn('github.com/crda/test/package1', vuln)
self.assertIsInstance(vuln.get('github.com/crda/test/package1'), list)
self.assertEqual(len(vuln.get('github.com/crda/test/package1')), 1)