def update_endpoint_status(existing_finding, new_finding, user):
# New endpoints are already added in serializers.py / views.py (see comment "# for existing findings: make sure endpoints are present or created")
# So we only need to mitigate endpoints that are no longer present
existing_finding_endpoint_status_list = existing_finding.endpoint_status.all(
)
new_finding_endpoints_list = new_finding.unsaved_endpoints
endpoint_status_to_mitigate = list(
filter(
lambda existing_finding_endpoint_status:
existing_finding_endpoint_status.endpoint not in
new_finding_endpoints_list, existing_finding_endpoint_status_list))
# Determine if this can be run async
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(endpoint_status_to_mitigate)
# If there is only one chunk, then do not bother with async
if len(chunk_list) < 2:
mitigate_endpoint_status(endpoint_status_to_mitigate,
user,
kwuser=user,
sync=True)
return
# First kick off all the workers
for endpoint_status_list in chunk_list:
mitigate_endpoint_status(endpoint_status_list,
user,
kwuser=user,
sync=False)
else:
mitigate_endpoint_status(endpoint_status_to_mitigate,
user,
kwuser=user,
sync=True)
def update_endpoint_status(existing_finding, new_finding, user):
# New endpoints are already added in serializers.py / views.py (see comment "# for existing findings: make sure endpoints are present or created")
# So we only need to mitigate endpoints that are no longer present
# using `.all()` will mark as mitigated also `endpoint_status` with flags `false_positive`, `out_of_scope` and `risk_accepted`. This is a known issue. This is not a bug. This is a future.
existing_finding_endpoint_status_list = existing_finding.endpoint_status.all()
new_finding_endpoints_list = new_finding.unsaved_endpoints
endpoint_status_to_mitigate = list(
filter(
lambda existing_finding_endpoint_status: existing_finding_endpoint_status.endpoint not in new_finding_endpoints_list,
existing_finding_endpoint_status_list)
)
# Determine if this can be run async
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(endpoint_status_to_mitigate)
# If there is only one chunk, then do not bother with async
if len(chunk_list) < 2:
mitigate_endpoint_status(endpoint_status_to_mitigate, user, kwuser=user, sync=True)
return
# First kick off all the workers
for endpoint_status_list in chunk_list:
mitigate_endpoint_status(endpoint_status_list, user, kwuser=user, sync=False)
else:
mitigate_endpoint_status(endpoint_status_to_mitigate, user, kwuser=user, sync=True)
def reimport_scan(self,
scan,
scan_type,
test,
active=True,
verified=True,
tags=None,
minimum_severity=None,
user=None,
endpoints_to_add=None,
scan_date=None,
version=None,
branch_tag=None,
build_id=None,
commit_hash=None,
push_to_jira=None,
close_old_findings=True,
group_by=None,
api_scan_configuration=None,
service=None):
logger.debug(f'REIMPORT_SCAN: parameters: {locals()}')
user = user or get_current_user()
now = timezone.now()
if api_scan_configuration:
if api_scan_configuration.product != test.engagement.product:
raise ValidationError(
'API Scan Configuration has to be from same product as the Test'
)
if test.api_scan_configuration != api_scan_configuration:
test.api_scan_configuration = api_scan_configuration
test.save()
# check if the parser that handle the scan_type manage tests
parser = get_parser(scan_type)
if hasattr(parser, 'get_tests'):
logger.debug('REIMPORT_SCAN parser v2: Create parse findings')
tests = parser.get_tests(scan_type, scan)
# for now we only consider the first test in the list and artificially aggregate all findings of all tests
# this is the same as the old behavior as current import/reimporter implementation doesn't handle the case
# when there is more than 1 test
parsed_findings = []
for test_raw in tests:
parsed_findings.extend(test_raw.findings)
else:
logger.debug('REIMPORT_SCAN: Parse findings')
parsed_findings = parser.get_findings(scan, test)
logger.debug('REIMPORT_SCAN: Processing findings')
new_findings = []
reactivated_findings = []
findings_to_mitigate = []
untouched_findings = []
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(parsed_findings)
results_list = []
# First kick off all the workers
for findings_list in chunk_list:
result = self.process_parsed_findings(
test,
findings_list,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service,
scan_date=scan_date,
sync=False)
# Since I dont want to wait until the task is done right now, save the id
# So I can check on the task later
results_list += [result]
# After all tasks have been started, time to pull the results
logger.debug('REIMPORT_SCAN: Collecting Findings')
for results in results_list:
serial_new_findings, serial_reactivated_findings, serial_findings_to_mitigate, serial_untouched_findings = results.get(
)
new_findings += [
next(serializers.deserialize("json", finding)).object
for finding in serial_new_findings
]
reactivated_findings += [
next(serializers.deserialize("json", finding)).object
for finding in serial_reactivated_findings
]
findings_to_mitigate += [
next(serializers.deserialize("json", finding)).object
for finding in serial_findings_to_mitigate
]
untouched_findings += [
next(serializers.deserialize("json", finding)).object
for finding in serial_untouched_findings
]
logger.debug('REIMPORT_SCAN: All Findings Collected')
# Indicate that the test is not complete yet as endpoints will still be rolling in.
test.percent_complete = 50
test.save()
importer_utils.update_test_progress(test)
else:
new_findings, reactivated_findings, findings_to_mitigate, untouched_findings = \
self.process_parsed_findings(test, parsed_findings, scan_type, user, active, verified,
minimum_severity=minimum_severity, endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira, group_by=group_by, now=now, service=service, scan_date=scan_date, sync=True)
closed_findings = []
if close_old_findings:
logger.debug(
'REIMPORT_SCAN: Closing findings no longer present in scan report'
)
closed_findings = self.close_old_findings(
test,
findings_to_mitigate,
scan_date,
user=user,
push_to_jira=push_to_jira)
logger.debug('REIMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, version, branch_tag, build_id,
commit_hash, now, scan_date)
if settings.TRACK_IMPORT_HISTORY:
logger.debug('REIMPORT_SCAN: Updating Import History')
importer_utils.update_import_history(
Test_Import.REIMPORT_TYPE, active, verified, tags,
minimum_severity, endpoints_to_add, version, branch_tag,
build_id, commit_hash, push_to_jira, close_old_findings, test,
new_findings, closed_findings, reactivated_findings)
logger.debug('REIMPORT_SCAN: Generating notifications')
updated_count = len(closed_findings) + len(reactivated_findings) + len(
new_findings)
if updated_count > 0:
notifications_helper.notify_scan_added(
test,
updated_count,
new_findings=new_findings,
findings_mitigated=closed_findings,
findings_reactivated=reactivated_findings,
findings_untouched=untouched_findings)
logger.debug('REIMPORT_SCAN: Done')
return test, updated_count, len(new_findings), len(
closed_findings), len(reactivated_findings), len(
untouched_findings)
def process_parsed_findings(self,
test,
parsed_findings,
scan_type,
user,
active,
verified,
minimum_severity=None,
endpoints_to_add=None,
push_to_jira=None,
group_by=None,
now=timezone.now(),
service=None,
scan_date=None,
**kwargs):
items = parsed_findings
original_items = list(test.finding_set.all())
new_items = []
mitigated_count = 0
finding_count = 0
finding_added_count = 0
reactivated_count = 0
reactivated_items = []
unchanged_count = 0
unchanged_items = []
logger.debug('starting reimport of %i items.',
len(items) if items else 0)
from dojo.importers.reimporter.utils import (
get_deduplication_algorithm_from_conf,
match_new_finding_to_existing_finding, update_endpoint_status,
reactivate_endpoint_status)
deduplication_algorithm = get_deduplication_algorithm_from_conf(
scan_type)
i = 0
logger.debug(
'STEP 1: looping over findings from the reimported report and trying to match them to existing findings'
)
deduplicationLogger.debug(
'Algorithm used for matching new findings to existing findings: %s',
deduplication_algorithm)
for item in items:
# FIXME hack to remove when all parsers have unit tests for this attribute
if item.severity.lower().startswith(
'info') and item.severity != 'Info':
item.severity = 'Info'
item.numerical_severity = Finding.get_numerical_severity(
item.severity)
if minimum_severity and (Finding.SEVERITIES[item.severity] >
Finding.SEVERITIES[minimum_severity]):
# finding's severity is below the configured threshold : ignoring the finding
continue
# existing findings may be from before we had component_name/version fields
component_name = item.component_name if hasattr(
item, 'component_name') else None
component_version = item.component_version if hasattr(
item, 'component_version') else None
if not hasattr(item, 'test'):
item.test = test
item.service = service
item.hash_code = item.compute_hash_code()
deduplicationLogger.debug("item's hash_code: %s", item.hash_code)
findings = match_new_finding_to_existing_finding(
item, test, deduplication_algorithm, scan_type)
deduplicationLogger.debug(
'found %i findings matching with current new finding',
len(findings))
if findings:
# existing finding found
finding = findings[0]
if finding.false_p or finding.out_of_scope or finding.risk_accepted:
logger.debug(
'%i: skipping existing finding (it is marked as false positive:%s and/or out of scope:%s or is a risk accepted:%s): %i:%s:%s:%s',
i, finding.false_p, finding.out_of_scope,
finding.risk_accepted, finding.id, finding,
finding.component_name, finding.component_version)
elif finding.mitigated or finding.is_mitigated:
logger.debug('%i: reactivating: %i:%s:%s:%s', i,
finding.id, finding, finding.component_name,
finding.component_version)
finding.mitigated = None
finding.is_mitigated = False
finding.mitigated_by = None
finding.active = True
finding.verified = verified
# existing findings may be from before we had component_name/version fields
finding.component_name = finding.component_name if finding.component_name else component_name
finding.component_version = finding.component_version if finding.component_version else component_version
# don't dedupe before endpoints are added
finding.save(dedupe_option=False)
note = Notes(entry="Re-activated by %s re-upload." %
scan_type,
author=user)
note.save()
endpoint_statuses = finding.endpoint_status.all()
# Determine if this can be run async
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(
endpoint_statuses)
# If there is only one chunk, then do not bother with async
if len(chunk_list) < 2:
reactivate_endpoint_status(endpoint_statuses,
sync=True)
logger.debug('IMPORT_SCAN: Split endpoints into ' +
str(len(chunk_list)) + ' chunks of ' +
str(chunk_list[0]))
# First kick off all the workers
for endpoint_status_list in chunk_list:
reactivate_endpoint_status(endpoint_status_list,
sync=False)
else:
reactivate_endpoint_status(endpoint_statuses,
sync=True)
finding.notes.add(note)
reactivated_items.append(finding)
reactivated_count += 1
else:
# existing findings may be from before we had component_name/version fields
logger.debug('%i: updating existing finding: %i:%s:%s:%s',
i, finding.id, finding,
finding.component_name,
finding.component_version)
if not finding.component_name or not finding.component_version:
finding.component_name = finding.component_name if finding.component_name else component_name
finding.component_version = finding.component_version if finding.component_version else component_version
finding.save(dedupe_option=False)
unchanged_items.append(finding)
unchanged_count += 1
if finding.dynamic_finding:
logger.debug(
"Re-import found an existing dynamic finding for this new finding. Checking the status of endpoints"
)
update_endpoint_status(finding, item, user)
else:
# no existing finding found
item.reporter = user
item.last_reviewed = timezone.now()
item.last_reviewed_by = user
item.verified = verified
item.active = active
# if scan_date was provided, override value from parser
if scan_date:
item.date = scan_date
# Save it. Don't dedupe before endpoints are added.
item.save(dedupe_option=False)
logger.debug(
'%i: reimport created new finding as no existing finding match: %i:%s:%s:%s',
i, item.id, item, item.component_name,
item.component_version)
# only new items get auto grouped to avoid confusion around already existing items that are already grouped
if settings.FEATURE_FINDING_GROUPS and group_by:
finding_helper.add_finding_to_auto_group(item, group_by)
finding_added_count += 1
new_items.append(item)
finding = item
if hasattr(item, 'unsaved_req_resp'):
for req_resp in item.unsaved_req_resp:
burp_rr = BurpRawRequestResponse(
finding=finding,
burpRequestBase64=base64.b64encode(
req_resp["req"].encode("utf-8")),
burpResponseBase64=base64.b64encode(
req_resp["resp"].encode("utf-8")))
burp_rr.clean()
burp_rr.save()
if item.unsaved_request and item.unsaved_response:
burp_rr = BurpRawRequestResponse(
finding=finding,
burpRequestBase64=base64.b64encode(
item.unsaved_request.encode()),
burpResponseBase64=base64.b64encode(
item.unsaved_response.encode()))
burp_rr.clean()
burp_rr.save()
# for existing findings: make sure endpoints are present or created
if finding:
finding_count += 1
if settings.ASYNC_FINDING_IMPORT:
importer_utils.chunk_endpoints_and_disperse(
finding, test, item.unsaved_endpoints)
else:
importer_utils.add_endpoints_to_unsaved_finding(
finding, test, item.unsaved_endpoints, sync=True)
if endpoints_to_add:
if settings.ASYNC_FINDING_IMPORT:
importer_utils.chunk_endpoints_and_disperse(
finding, test, endpoints_to_add)
else:
importer_utils.add_endpoints_to_unsaved_finding(
finding, test, endpoints_to_add, sync=True)
if item.unsaved_tags:
finding.tags = item.unsaved_tags
if item.unsaved_files:
for unsaved_file in item.unsaved_files:
data = base64.b64decode(unsaved_file.get('data'))
title = unsaved_file.get('title', '<No title>')
file_upload, file_upload_created = FileUpload.objects.get_or_create(
title=title, )
file_upload.file.save(title, ContentFile(data))
file_upload.save()
finding.files.add(file_upload)
# existing findings may be from before we had component_name/version fields
finding.component_name = finding.component_name if finding.component_name else component_name
finding.component_version = finding.component_version if finding.component_version else component_version
# finding = new finding or existing finding still in the upload report
# to avoid pushing a finding group multiple times, we push those outside of the loop
if settings.FEATURE_FINDING_GROUPS and finding.finding_group:
finding.save()
else:
finding.save(push_to_jira=push_to_jira)
to_mitigate = set(original_items) - set(reactivated_items) - set(
unchanged_items)
untouched = set(unchanged_items) - set(to_mitigate)
if settings.FEATURE_FINDING_GROUPS and push_to_jira:
for finding_group in set([
finding.finding_group for finding in reactivated_items +
unchanged_items + new_items
if finding.finding_group is not None
]):
jira_helper.push_to_jira(finding_group)
sync = kwargs.get('sync', False)
if not sync:
serialized_new_items = [
serializers.serialize('json', [
finding,
]) for finding in new_items
]
serialized_reactivated_items = [
serializers.serialize('json', [
finding,
]) for finding in reactivated_items
]
serialized_to_mitigate = [
serializers.serialize('json', [
finding,
]) for finding in to_mitigate
]
serialized_untouched = [
serializers.serialize('json', [
finding,
]) for finding in untouched
]
return serialized_new_items, serialized_reactivated_items, serialized_to_mitigate, serialized_untouched
return new_items, reactivated_items, to_mitigate, untouched
def import_scan(self,
scan,
scan_type,
engagement,
lead,
environment,
active,
verified,
tags=None,
minimum_severity=None,
user=None,
endpoints_to_add=None,
scan_date=None,
version=None,
branch_tag=None,
build_id=None,
commit_hash=None,
push_to_jira=None,
close_old_findings=False,
group_by=None,
api_scan_configuration=None,
service=None,
title=None):
logger.debug(f'IMPORT_SCAN: parameters: {locals()}')
user = user or get_current_user()
now = timezone.now()
if api_scan_configuration and api_scan_configuration.product != engagement.product:
raise ValidationError(
'API Scan Configuration has to be from same product as the Engagement'
)
# check if the parser that handle the scan_type manage tests
# if yes, we parse the data first
# after that we customize the Test_Type to reflect the data
# This allow us to support some meta-formats like SARIF or the generic format
parser = get_parser(scan_type)
if hasattr(parser, 'get_tests'):
logger.debug(
'IMPORT_SCAN parser v2: Create Test and parse findings')
tests = parser.get_tests(scan_type, scan)
# for now we only consider the first test in the list and artificially aggregate all findings of all tests
# this is the same as the old behavior as current import/reimporter implementation doesn't handle the case
# when there is more than 1 test
#
# we also aggregate the label of the Test_type to show the user the original scan_type
# only if they are different. This is to support meta format like SARIF
# so a report that have the label 'CodeScanner' will be changed to 'CodeScanner Scan (SARIF)'
test_type_name = scan_type
if len(tests) > 0:
if tests[0].type:
test_type_name = tests[0].type + " Scan"
if test_type_name != scan_type:
test_type_name = f"{test_type_name} ({scan_type})"
test = self.create_test(
scan_type,
test_type_name,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration,
title=title)
# This part change the name of the Test
# we get it from the data of the parser
test_raw = tests[0]
if test_raw.name:
test.name = test_raw.name
if test_raw.description:
test.description = test_raw.description
test.save()
logger.debug(
'IMPORT_SCAN parser v2: Parse findings (aggregate)')
# currently we only support import one Test
# so for parser that support multiple tests (like SARIF)
# we aggregate all the findings into one uniq test
parsed_findings = []
for test_raw in tests:
parsed_findings.extend(test_raw.findings)
else:
logger.info(f'No tests found in import for {scan_type}')
else:
logger.debug('IMPORT_SCAN: Create Test')
# by default test_type == scan_type
test = self.create_test(
scan_type,
scan_type,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration,
title=title)
logger.debug('IMPORT_SCAN: Parse findings')
parser = get_parser(scan_type)
parsed_findings = parser.get_findings(scan, test)
logger.debug('IMPORT_SCAN: Processing findings')
new_findings = []
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(parsed_findings)
results_list = []
# First kick off all the workers
for findings_list in chunk_list:
result = self.process_parsed_findings(
test,
findings_list,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service,
scan_date=scan_date,
sync=False)
# Since I dont want to wait until the task is done right now, save the id
# So I can check on the task later
results_list += [result]
# After all tasks have been started, time to pull the results
logger.info('IMPORT_SCAN: Collecting Findings')
for results in results_list:
serial_new_findings = results.get()
new_findings += [
next(serializers.deserialize("json", finding)).object
for finding in serial_new_findings
]
logger.info('IMPORT_SCAN: All Findings Collected')
# Indicate that the test is not complete yet as endpoints will still be rolling in.
test.percent_complete = 50
test.save()
importer_utils.update_test_progress(test)
else:
new_findings = self.process_parsed_findings(
test,
parsed_findings,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service,
scan_date=scan_date,
sync=True)
closed_findings = []
if close_old_findings:
logger.debug(
'IMPORT_SCAN: Closing findings no longer present in scan report'
)
closed_findings = self.close_old_findings(
test,
scan_date,
user=user,
push_to_jira=push_to_jira,
service=service)
logger.debug('IMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, version, branch_tag, build_id,
commit_hash, now, scan_date)
if settings.TRACK_IMPORT_HISTORY:
logger.debug('IMPORT_SCAN: Updating Import History')
importer_utils.update_import_history(
Test_Import.IMPORT_TYPE, active, verified, tags,
minimum_severity, endpoints_to_add, version, branch_tag,
build_id, commit_hash, push_to_jira, close_old_findings, test,
new_findings, closed_findings)
logger.debug('IMPORT_SCAN: Generating notifications')
notifications_helper.notify_test_created(test)
updated_count = len(new_findings) + len(closed_findings)
if updated_count > 0:
notifications_helper.notify_scan_added(
test,
updated_count,
new_findings=new_findings,
findings_mitigated=closed_findings)
logger.debug('IMPORT_SCAN: Done')
return test, len(new_findings), len(closed_findings)