def _auto_triage_all(
self,
analysis_result: AnalysisResult,
):
for component in analysis_result.components():
for vulnerability in component.vulnerabilities():
if (vulnerability.cve_severity() >= self.cvss_threshold and not
vulnerability.historical()) and not vulnerability.has_triage():
Shared.add_triage(
protecode_client=self._api,
component_name=component.name(),
component_version=component.version(),
product_id=analysis_result.product_id(),
vulnerability_cve=vulnerability.cve(),
description='Auto-generated due to label skip-scan',
extended_objects=component.extended_objects(),
)
def _determine_upload_action(
self,
container_image: ContainerImage,
scan_result: AnalysisResult,
):
check_type(container_image, ContainerImage)
# take shortcut if 'force upload' is configured.
if self._processing_mode is ProcessingMode.FORCE_UPLOAD:
return UploadAction.UPLOAD
if self._processing_mode in (
ProcessingMode.UPLOAD_IF_CHANGED,
ProcessingMode.RESCAN,
):
# if no scan_result is available, we have to upload in all remaining cases
if not scan_result:
return UploadAction.UPLOAD
# determine if image to be uploaded is already present in protecode
metadata = scan_result.custom_data()
image_reference = metadata.get('IMAGE_REFERENCE')
image_changed = image_reference != container_image.image_reference()
if image_changed:
return UploadAction.UPLOAD
if self._processing_mode is ProcessingMode.UPLOAD_IF_CHANGED:
return UploadAction.SKIP
elif self._processing_mode is ProcessingMode.RESCAN:
# Wait for the current scan to finish if it there is still one pending
if scan_result.status() is ProcessingStatus.BUSY:
return UploadAction.WAIT_FOR_RESULT
short_scan_result = self._api.scan_result_short(
scan_result.product_id())
if short_scan_result.is_stale():
if not short_scan_result.has_binary():
return UploadAction.UPLOAD
else:
return UploadAction.RESCAN
else:
return UploadAction.SKIP
else:
raise NotImplementedError
def _determine_upload_action(
self,
container_image: ContainerImage,
scan_result: AnalysisResult,
):
check_type(container_image, ContainerImage)
if self._processing_mode in (
ProcessingMode.UPLOAD_IF_CHANGED,
ProcessingMode.RESCAN,
ProcessingMode.FORCE_UPLOAD,
):
# if no scan_result is available, we have to upload in all cases
if not scan_result:
return UploadAction.UPLOAD
# determine if image to be uploaded is already present in protecode
metadata = scan_result.custom_data()
image_reference = metadata.get('IMAGE_REFERENCE')
image_changed = image_reference != container_image.image_reference()
if image_changed:
return UploadAction.UPLOAD
if self._processing_mode is ProcessingMode.UPLOAD_IF_CHANGED:
return UploadAction.SKIP
elif self._processing_mode is ProcessingMode.RESCAN:
short_scan_result = self._api.scan_result_short(
scan_result.product_id())
if short_scan_result.is_stale():
if not short_scan_result.has_binary():
return UploadAction.UPLOAD
else:
return UploadAction.RESCAN
else:
return UploadAction.SKIP
elif self._processing_mode is ProcessingMode.FORCE_UPLOAD:
return UploadAction.UPLOAD
else:
raise NotImplementedError
def _iter_matching_products(url: str):
res = self._get(url=url)
res.raise_for_status()
res = res.json()
products: list[dict] = res['products']
for product in products:
if not full_match(product.get('custom_data')):
continue
yield AnalysisResult(product)
if next_page_url := res.get('next'):
yield from _iter_matching_products(url=next_page_url)
def _import_triages_from_gcr(
self,
scan_result: AnalysisResult
) -> AnalysisResult:
image_ref = scan_result.custom_data().get('IMAGE_REFERENCE', None)
scan_result_triages = list(self._existing_triages([scan_result]))
if not image_ref:
logging.warning(f'no image-ref-name custom-prop for {scan_result.product_id()}')
return scan_result
gcr_scanner = GcrSynchronizer(image_ref, self.cvss_threshold, self._api)
return gcr_scanner.sync(scan_result, scan_result_triages)
def upload(
self,
application_name: str,
group_id: str,
data: typing.Generator[bytes, None, None],
replace_id: int = None,
custom_attribs={},
) -> AnalysisResult:
url = self._routes.upload(file_name=application_name)
headers = {'Group': str(group_id)}
if replace_id:
headers['Replace'] = str(replace_id)
headers.update(self._metadata_dict(custom_attribs))
result = self._put(
url=url,
headers=headers,
data=data,
)
return AnalysisResult(raw_dict=result.json().get('results'))
def _import_triages_from_gcr(self, scan_result: AnalysisResult):
image_ref = scan_result.custom_data().get('IMAGE_REFERENCE', None)
scan_result_triages = list(self._existing_triages([scan_result]))
if not image_ref:
logging.warning(
f'no image-ref-name custom-prop for {scan_result.product_id()}'
)
return scan_result
grafeas_client = ccc.gcp.GrafeasClient.for_image(image_ref)
if not grafeas_client.scan_available(image_reference=image_ref):
ci.util.warning(f'no scan result available in gcr: {image_ref}')
return scan_result
# determine worst CVE according to GCR's data
worst_cvss = -1
worst_effective_vuln = Severity.SEVERITY_UNSPECIFIED
try:
vulnerabilities_from_grafeas = list(
grafeas_client.filter_vulnerabilities(
image_reference=image_ref,
cvss_threshold=self.cvss_threshold,
))
for gcr_occ in vulnerabilities_from_grafeas:
gcr_score = gcr_occ.vulnerability.cvssScore
worst_cvss = max(worst_cvss, gcr_score)
effective_sev = gcr_occ.vulnerability.effectiveSeverity
worst_effective_vuln = max(worst_effective_vuln, effective_sev)
except ccc.gcp.VulnerabilitiesRetrievalFailed as vrf:
ci.util.warning(str(vrf))
# warn, but ignore
return scan_result
if worst_cvss >= self.cvss_threshold:
ci.util.info(
f'GCR\'s worst CVSS rating is above threshold: {worst_cvss}')
ci.util.info(f'however, consider: {worst_effective_vuln=}')
triage_remainder = False
else:
# worst finding below our threshold -> we may safely triage everything
# w/o being able to match triages component-wise
triage_remainder = True
def find_worst_vuln(component, vulnerability, grafeas_vulns):
component_name = component.name()
cve_str = vulnerability.cve()
worst_cve = -1
worst_effective_severity = Severity.SEVERITY_UNSPECIFIED
found_it = False
for gv in grafeas_vulns:
v = gv.vulnerability
if v.shortDescription != cve_str: # TODO: could also check the note name
continue
for pi in v.packageIssue:
v_name = pi.affectedPackage
if not v_name == component_name:
# XXX maybe we should be a bit more defensive, and check for CVE equality
# (if CVEs match, but compont name differs, a human could/should have a look)
if v.shortDescription == cve_str:
ci.util.warning(
f'XXX check if this is a match: {v_name} / {component_name}'
)
continue
found_it = True
# XXX should also check for version
worst_cve = max(worst_cve, v.cvssScore)
worst_effective_severity = max(worst_effective_severity,
v.effectiveSeverity)
return found_it, worst_cve, worst_effective_severity
def find_component_version(component_name, occurrences):
determined_version = None
for occurrence in occurrences:
package_issues = occurrence.vulnerability.packageIssue
for package_issue in package_issues:
package_name = package_issue.affectedPackage
if package_name == component_name:
if (determined_version is not None
and determined_version !=
package_issue.affectedVersion.fullName):
# found more than one possible version. Return None since we cannot
# be sure which version is correct
return None
determined_version = package_issue.affectedVersion.fullName
return determined_version
# if this line is reached, all vulnerabilities are considered to be less severe than
# protecode thinks. So triage all of them away
components_count = 0
vulnerabilities_count = 0 # only above threshold, and untriaged
skipped_due_to_historicalness = 0
skipped_due_to_existing_triages = 0
triaged_due_to_max_count = 0
triaged_due_to_gcr_optimism = 0
triaged_due_to_absent_count = 0
# helper functon to avoid duplicating triages later
def _triage_already_present(triage_dict, triages):
for triage in triages:
if triage.vulnerability_id() != triage_dict['vulns'][0]:
continue
if triage.component_name() != triage_dict['component']:
continue
if triage.description() != triage_dict['description']:
continue
return True
return False
for component in scan_result.components():
components_count += 1
version = component.version()
component_name = component.name()
if not version:
# if version is not known to protecode, see if it can be determined using gcr's
# vulnerability-assessment
if (version := find_component_version(
component_name,
vulnerabilities_from_grafeas)): # noqa:E203,E501
ci.util.info(
f"Grafeas has version '{version}' for undetermined protecode-component "
f"'{component_name}'. Will try to import version to Protecode."
)
try:
self._api.set_component_version(
component_name=component_name,
component_version=version,
objects=[
o.sha1() for o in component.extended_objects()
],
scope=VersionOverrideScope.APP,
app_id=scan_result.product_id(),
)
except requests.exceptions.HTTPError as http_err:
ci.util.warning(
f"Unable to set version for component '{component_name}': {http_err}."
)
for vulnerability in component.vulnerabilities():
vulnerabilities_count += 1
severity = float(
vulnerability.cve_severity_str(
protecode.model.CVSSVersion.V3))
if severity < self.cvss_threshold:
continue # only triage vulnerabilities above threshold
if vulnerability.has_triage():
skipped_due_to_existing_triages += 1
continue # nothing to do
if vulnerability.historical():
skipped_due_to_historicalness += 1
continue # historical vulnerabilities cannot be triaged.
if not version:
# Protecode only allows triages for components with known version.
# set version to be able to triage away.
version = '[ci]-not-found-in-GCR'
ci.util.info(
f"Setting dummy version for component '{component_name}'"
)
try:
self._api.set_component_version(
component_name=component_name,
component_version=version,
objects=[
o.sha1() for o in component.extended_objects()
],
scope=VersionOverrideScope.APP,
app_id=scan_result.product_id(),
)
except requests.exceptions.HTTPError as http_err:
ci.util.warning(
f"Unable to set version for component '{component_name}': {http_err}."
)
# version was not set - cannot triage
continue
if not triage_remainder:
found_it, worst_cve, worst_eff = find_worst_vuln(
component=component,
vulnerability=vulnerability,
grafeas_vulns=vulnerabilities_from_grafeas,
)
if not found_it:
ci.util.info(
f'did not find {component.name()}:{vulnerability.cve()} in GCR'
)
triaged_due_to_absent_count += 1
description = \
'[ci] vulnerability was not reported by GCR'
elif worst_cve >= self.cvss_threshold:
triaged_due_to_gcr_optimism += 1
ci.util.info(
f'found {component.name()}, but is above threshold {worst_cve=}'
)
continue
else:
description = \
f'[ci] vulnerability was assessed by GCR with {worst_cve}'
else:
triaged_due_to_max_count += 1
description = \
'[ci] vulnerability was not found by GCR'
triage_dict = {
'component': component.name(),
'version': version,
'vulns': [vulnerability.cve()],
'scope': protecode.model.TriageScope.RESULT.value,
'reason': 'OT', # "other"
'description': description,
'product_id': scan_result.product_id(),
}
if _triage_already_present(triage_dict, scan_result_triages):
ci.util.info(
f'triage {component.name()}:{vulnerability.cve()} already present.'
)
continue
try:
self._api.add_triage_raw(triage_dict=triage_dict)
ci.util.info(
f'added triage: {component.name()}:{vulnerability.cve()}'
)
except requests.exceptions.HTTPError as http_err:
# since we are auto-importing anyway, be a bit tolerant
ci.util.warning(f'failed to add triage: {http_err}')
def scan_result(self, product_id: int) -> AnalysisResult:
url = self._routes.product(product_id=product_id)
result = self._get(url=url, ).json()['results']
return AnalysisResult(raw_dict=result)
def sync(
self,
scan_result: AnalysisResult,
scan_result_triages: Iterable[pm.Triage]
) -> AnalysisResult:
if not self.grafeas_client.scan_available(image_reference=self.image_ref):
logger.warning(f'no scan result available in gcr: {self.image_ref}')
return scan_result
# determine worst CVE according to GCR's data
worst_cvss = -1
worst_effective_vuln = Severity.SEVERITY_UNSPECIFIED
try:
vulnerabilities_from_grafeas = list(
self.grafeas_client.filter_vulnerabilities(
image_reference=self.image_ref,
cvss_threshold=self.cvss_threshold,
)
)
for gcr_occ in vulnerabilities_from_grafeas:
gcr_score = gcr_occ.vulnerability.cvssScore
worst_cvss = max(worst_cvss, gcr_score)
effective_sev = gcr_occ.vulnerability.effectiveSeverity
worst_effective_vuln = max(worst_effective_vuln, effective_sev)
except ccc.gcp.VulnerabilitiesRetrievalFailed as vrf:
logger.warning(str(vrf))
# warn, but ignore
return scan_result
if worst_cvss >= self.cvss_threshold:
logger.info(f'GCR\'s worst CVSS rating is above threshold: {worst_cvss}')
logger.info(f'however, consider: {worst_effective_vuln=} ({scan_result.product_id()})')
triage_remainder = False
else:
# worst finding below our threshold -> we may safely triage everything
# w/o being able to match triages component-wise
triage_remainder = True
# if this line is reached, all vulnerabilities are considered to be less severe than
# protecode thinks. So triage all of them away
components_count = 0
vulnerabilities_count = 0 # only above threshold, and untriaged
skipped_due_to_historicalness = 0
skipped_due_to_existing_triages = 0
triaged_due_to_max_count = 0
triaged_due_to_gcr_optimism = 0
triaged_due_to_absent_count = 0
for component in scan_result.components():
components_count += 1
for vulnerability in component.vulnerabilities():
vulnerabilities_count += 1
severity = float(vulnerability.cve_severity_str(pm.CVSSVersion.V3))
if severity < self.cvss_threshold:
continue # only triage vulnerabilities above threshold
if vulnerability.has_triage():
skipped_due_to_existing_triages += 1
continue # nothing to do
if vulnerability.historical():
skipped_due_to_historicalness += 1
continue # historical vulnerabilities cannot be triaged.
if not triage_remainder:
found_it, worst_cve, worst_eff = self._find_worst_vuln(
component=component,
vulnerability=vulnerability,
grafeas_vulns=vulnerabilities_from_grafeas,
)
if not found_it:
logger.info(
f'did not find {component.name()}:{vulnerability.cve()} in GCR'
)
triaged_due_to_absent_count += 1
description = \
'[ci] vulnerability was not reported by GCR'
elif worst_cve >= self.cvss_threshold:
triaged_due_to_gcr_optimism += 1
logger.info(
f'found {component.name()}, but is above threshold {worst_cve=}'
)
continue
else:
description = \
f'[ci] vulnerability was assessed by GCR with {worst_cve}'
else:
triaged_due_to_max_count += 1
description = \
'[ci] vulnerability was not found by GCR'
if self._triage_already_present(
vulnerability_id=vulnerability.cve(),
component_name=component.name(),
description=description,
triages=scan_result_triages,
):
logger.info(f'triage {component.name()}:{vulnerability.cve()} already present.')
continue
Shared.add_triage(
protecode_client=self.protecode_client,
component_name=component.name(),
component_version=component.version(),
product_id=scan_result.product_id(),
vulnerability_cve=vulnerability.cve(),
description=description,
extended_objects=component.extended_objects(),
)
logger.info(textwrap.dedent(f'''
Product: {scan_result.display_name()} (ID: {scan_result.product_id()})
Statistics: {components_count=} {vulnerabilities_count=}
{skipped_due_to_historicalness=} {skipped_due_to_existing_triages=}
{triaged_due_to_max_count=} {triaged_due_to_gcr_optimism=}
{triaged_due_to_absent_count=}
'''
))
# retrieve scan-results again to get filtered results after taking triages into account
return self.protecode_client.scan_result(product_id=scan_result.product_id())