def import_scan(self, scan, scan_type, engagement, lead, environment, active, verified, tags=None, minimum_severity=None,
user=None, endpoints_to_add=None, scan_date=None, version=None, branch_tag=None, build_id=None,
commit_hash=None, push_to_jira=None, close_old_findings=False):
user = user or get_current_user()
now = timezone.now()
# retain weird existing logic to use current time for provided scan date
scan_date_time = datetime.datetime.combine(scan_date, timezone.now().time())
if settings.USE_TZ:
scan_date_time = timezone.make_aware(scan_date_time, timezone.get_default_timezone())
logger.debug('IMPORT_SCAN: Create Test')
test = self.create_test(scan_type, engagement, lead, environment, scan_date=scan_date, tags=tags,
version=version, branch_tag=branch_tag, build_id=build_id, commit_hash=commit_hash, now=now)
logger.debug('IMPORT_SCAN: Parse findings')
parsed_findings = importer_utils.parse_findings(scan, test, active, verified, scan_type)
logger.debug('IMPORT_SCAN: Processing findings')
new_findings = self.process_parsed_findings(test, parsed_findings, scan_type, user, active,
verified, minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add, push_to_jira=push_to_jira,
now=now)
closed_findings = []
if close_old_findings:
logger.debug('IMPORT_SCAN: Closing findings no longer present in scan report')
closed_findings = self.close_old_findings(test, scan_date_time, user=user, push_to_jira=push_to_jira)
logger.debug('IMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, scan_date, version, branch_tag, build_id, commit_hash, now, scan_date_time)
if settings.TRACK_IMPORT_HISTORY:
logger.debug('IMPORT_SCAN: Updating Import History')
importer_utils.update_import_history(Test_Import.IMPORT_TYPE, active, verified, tags, minimum_severity,
endpoints_to_add, version, branch_tag, build_id, commit_hash,
push_to_jira, close_old_findings, test, new_findings, closed_findings)
logger.debug('IMPORT_SCAN: Generating notifications')
notifications_helper.notify_test_created(test)
updated_count = len(new_findings) + len(closed_findings)
if updated_count > 0:
notifications_helper.notify_scan_added(test, updated_count, new_findings=new_findings, findings_mitigated=closed_findings)
logger.debug('IMPORT_SCAN: Done')
return test, len(new_findings), len(closed_findings)
def add_tests(request, eid):
eng = Engagement.objects.get(id=eid)
cred_form = CredMappingForm()
cred_form.fields["cred_user"].queryset = Cred_Mapping.objects.filter(
engagement=eng).order_by('cred_id')
if request.method == 'POST':
form = TestForm(request.POST, engagement=eng)
cred_form = CredMappingForm(request.POST)
cred_form.fields["cred_user"].queryset = Cred_Mapping.objects.filter(
engagement=eng).order_by('cred_id')
if form.is_valid():
new_test = form.save(commit=False)
# set default scan_type as it's used in reimport
new_test.scan_type = new_test.test_type.name
new_test.engagement = eng
try:
new_test.lead = User.objects.get(id=form['lead'].value())
except:
new_test.lead = None
pass
# Set status to in progress if a test is added
if eng.status != "In Progress" and eng.active is True:
eng.status = "In Progress"
eng.save()
new_test.save()
# Save the credential to the test
if cred_form.is_valid():
if cred_form.cleaned_data['cred_user']:
# Select the credential mapping object from the selected list and only allow if the credential is associated with the product
cred_user = Cred_Mapping.objects.filter(
pk=cred_form.cleaned_data['cred_user'].id,
engagement=eid).first()
new_f = cred_form.save(commit=False)
new_f.test = new_test
new_f.cred_id = cred_user.cred_id
new_f.save()
messages.add_message(
request,
messages.SUCCESS,
'Test added successfully.',
extra_tags='alert-success')
notifications_helper.notify_test_created(new_test)
if '_Add Another Test' in request.POST:
return HttpResponseRedirect(
reverse('add_tests', args=(eng.id, )))
elif '_Add Findings' in request.POST:
return HttpResponseRedirect(
reverse('add_findings', args=(new_test.id, )))
elif '_Finished' in request.POST:
return HttpResponseRedirect(
reverse('view_engagement', args=(eng.id, )))
else:
form = TestForm(engagement=eng)
form.initial['target_start'] = eng.target_start
form.initial['target_end'] = eng.target_end
form.initial['lead'] = request.user
add_breadcrumb(
parent=eng, title="Add Tests", top_level=False, request=request)
product_tab = Product_Tab(eng.product.id, title="Add Tests", tab="engagements")
product_tab.setEngagement(eng)
return render(request, 'dojo/add_tests.html', {
'product_tab': product_tab,
'form': form,
'cred_form': cred_form,
'eid': eid,
'eng': eng
})
def import_scan(self,
scan,
scan_type,
engagement,
lead,
environment,
active,
verified,
tags=None,
minimum_severity=None,
user=None,
endpoints_to_add=None,
scan_date=None,
version=None,
branch_tag=None,
build_id=None,
commit_hash=None,
push_to_jira=None,
close_old_findings=False,
group_by=None,
api_scan_configuration=None,
service=None,
title=None):
logger.debug(f'IMPORT_SCAN: parameters: {locals()}')
user = user or get_current_user()
now = timezone.now()
if api_scan_configuration and api_scan_configuration.product != engagement.product:
raise ValidationError(
'API Scan Configuration has to be from same product as the Engagement'
)
# check if the parser that handle the scan_type manage tests
# if yes, we parse the data first
# after that we customize the Test_Type to reflect the data
# This allow us to support some meta-formats like SARIF or the generic format
parser = get_parser(scan_type)
if hasattr(parser, 'get_tests'):
logger.debug(
'IMPORT_SCAN parser v2: Create Test and parse findings')
tests = parser.get_tests(scan_type, scan)
# for now we only consider the first test in the list and artificially aggregate all findings of all tests
# this is the same as the old behavior as current import/reimporter implementation doesn't handle the case
# when there is more than 1 test
#
# we also aggregate the label of the Test_type to show the user the original scan_type
# only if they are different. This is to support meta format like SARIF
# so a report that have the label 'CodeScanner' will be changed to 'CodeScanner Scan (SARIF)'
test_type_name = scan_type
if len(tests) > 0:
if tests[0].type:
test_type_name = tests[0].type + " Scan"
if test_type_name != scan_type:
test_type_name = f"{test_type_name} ({scan_type})"
test = self.create_test(
scan_type,
test_type_name,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration,
title=title)
# This part change the name of the Test
# we get it from the data of the parser
test_raw = tests[0]
if test_raw.name:
test.name = test_raw.name
if test_raw.description:
test.description = test_raw.description
test.save()
logger.debug(
'IMPORT_SCAN parser v2: Parse findings (aggregate)')
# currently we only support import one Test
# so for parser that support multiple tests (like SARIF)
# we aggregate all the findings into one uniq test
parsed_findings = []
for test_raw in tests:
parsed_findings.extend(test_raw.findings)
else:
logger.info(f'No tests found in import for {scan_type}')
else:
logger.debug('IMPORT_SCAN: Create Test')
# by default test_type == scan_type
test = self.create_test(
scan_type,
scan_type,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration,
title=title)
logger.debug('IMPORT_SCAN: Parse findings')
parser = get_parser(scan_type)
parsed_findings = parser.get_findings(scan, test)
logger.debug('IMPORT_SCAN: Processing findings')
new_findings = []
if settings.ASYNC_FINDING_IMPORT:
chunk_list = importer_utils.chunk_list(parsed_findings)
results_list = []
# First kick off all the workers
for findings_list in chunk_list:
result = self.process_parsed_findings(
test,
findings_list,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service,
scan_date=scan_date,
sync=False)
# Since I dont want to wait until the task is done right now, save the id
# So I can check on the task later
results_list += [result]
# After all tasks have been started, time to pull the results
logger.info('IMPORT_SCAN: Collecting Findings')
for results in results_list:
serial_new_findings = results.get()
new_findings += [
next(serializers.deserialize("json", finding)).object
for finding in serial_new_findings
]
logger.info('IMPORT_SCAN: All Findings Collected')
# Indicate that the test is not complete yet as endpoints will still be rolling in.
test.percent_complete = 50
test.save()
importer_utils.update_test_progress(test)
else:
new_findings = self.process_parsed_findings(
test,
parsed_findings,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service,
scan_date=scan_date,
sync=True)
closed_findings = []
if close_old_findings:
logger.debug(
'IMPORT_SCAN: Closing findings no longer present in scan report'
)
closed_findings = self.close_old_findings(
test,
scan_date,
user=user,
push_to_jira=push_to_jira,
service=service)
logger.debug('IMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, version, branch_tag, build_id,
commit_hash, now, scan_date)
if settings.TRACK_IMPORT_HISTORY:
logger.debug('IMPORT_SCAN: Updating Import History')
importer_utils.update_import_history(
Test_Import.IMPORT_TYPE, active, verified, tags,
minimum_severity, endpoints_to_add, version, branch_tag,
build_id, commit_hash, push_to_jira, close_old_findings, test,
new_findings, closed_findings)
logger.debug('IMPORT_SCAN: Generating notifications')
notifications_helper.notify_test_created(test)
updated_count = len(new_findings) + len(closed_findings)
if updated_count > 0:
notifications_helper.notify_scan_added(
test,
updated_count,
new_findings=new_findings,
findings_mitigated=closed_findings)
logger.debug('IMPORT_SCAN: Done')
return test, len(new_findings), len(closed_findings)
def import_scan(self,
scan,
scan_type,
engagement,
lead,
environment,
active,
verified,
tags=None,
minimum_severity=None,
user=None,
endpoints_to_add=None,
scan_date=None,
version=None,
branch_tag=None,
build_id=None,
commit_hash=None,
push_to_jira=None,
close_old_findings=False,
group_by=None,
api_scan_configuration=None,
service=None):
logger.debug(f'IMPORT_SCAN: parameters: {locals()}')
user = user or get_current_user()
now = timezone.now()
# retain weird existing logic to use current time for provided scan date
scan_date_time = datetime.datetime.combine(scan_date,
timezone.now().time())
if settings.USE_TZ:
scan_date_time = timezone.make_aware(
scan_date_time, timezone.get_default_timezone())
if api_scan_configuration and api_scan_configuration.product != engagement.product:
raise ValidationError(
'API Scan Configuration has to be from same product as the Engagement'
)
# check if the parser that handle the scan_type manage tests
# if yes, we parse the data first
# after that we customize the Test_Type to reflect the data
# This allow us to support some meta-formats like SARIF or the generic format
parser = get_parser(scan_type)
if hasattr(parser, 'get_tests'):
logger.debug(
'IMPORT_SCAN parser v2: Create Test and parse findings')
tests = parser.get_tests(scan_type, scan)
# for now we only consider the first test in the list and artificially aggregate all findings of all tests
# this is the same as the old behavior as current import/reimporter implementation doesn't handle the case
# when there is more than 1 test
#
# we also aggregate the label of the Test_type to show the user the original scan_type
# only if they are different. This is to support meta format like SARIF
# so a report that have the label 'CodeScanner' will be changed to 'CodeScanner Scan (SARIF)'
test_type_name = scan_type
if len(tests) > 0:
if tests[0].type:
test_type_name = tests[0].type + " Scan"
if test_type_name != scan_type:
test_type_name = f"{test_type_name} ({scan_type})"
test = self.create_test(
scan_type,
test_type_name,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration)
# This part change the name of the Test
# we get it from the data of the parser
test_raw = tests[0]
if test_raw.name:
test.name = test_raw.name
if test_raw.description:
test.description = test_raw.description
test.save()
logger.debug(
'IMPORT_SCAN parser v2: Parse findings (aggregate)')
# currently we only support import one Test
# so for parser that support multiple tests (like SARIF)
# we aggregate all the findings into one uniq test
parsed_findings = []
for test_raw in tests:
parsed_findings.extend(test_raw.findings)
else:
logger.info(f'No tests found in import for {scan_type}')
else:
logger.debug('IMPORT_SCAN: Create Test')
# by default test_type == scan_type
test = self.create_test(
scan_type,
scan_type,
engagement,
lead,
environment,
scan_date=scan_date,
tags=tags,
version=version,
branch_tag=branch_tag,
build_id=build_id,
commit_hash=commit_hash,
now=now,
api_scan_configuration=api_scan_configuration)
logger.debug('IMPORT_SCAN: Parse findings')
parser = get_parser(scan_type)
parsed_findings = parser.get_findings(scan, test)
logger.debug('IMPORT_SCAN: Processing findings')
new_findings = self.process_parsed_findings(
test,
parsed_findings,
scan_type,
user,
active,
verified,
minimum_severity=minimum_severity,
endpoints_to_add=endpoints_to_add,
push_to_jira=push_to_jira,
group_by=group_by,
now=now,
service=service)
closed_findings = []
if close_old_findings:
logger.debug(
'IMPORT_SCAN: Closing findings no longer present in scan report'
)
closed_findings = self.close_old_findings(
test, scan_date_time, user=user, push_to_jira=push_to_jira)
logger.debug('IMPORT_SCAN: Updating test/engagement timestamps')
importer_utils.update_timestamps(test, scan_date, version, branch_tag,
build_id, commit_hash, now,
scan_date_time)
if settings.TRACK_IMPORT_HISTORY:
logger.debug('IMPORT_SCAN: Updating Import History')
importer_utils.update_import_history(
Test_Import.IMPORT_TYPE, active, verified, tags,
minimum_severity, endpoints_to_add, version, branch_tag,
build_id, commit_hash, push_to_jira, close_old_findings, test,
new_findings, closed_findings)
logger.debug('IMPORT_SCAN: Generating notifications')
notifications_helper.notify_test_created(test)
updated_count = len(new_findings) + len(closed_findings)
if updated_count > 0:
notifications_helper.notify_scan_added(
test,
updated_count,
new_findings=new_findings,
findings_mitigated=closed_findings)
logger.debug('IMPORT_SCAN: Done')
return test, len(new_findings), len(closed_findings)