def create_scan(internal_scan_type: str,
urllist: UrlList,
manual_or_scheduled: str = "scheduled") -> int:
new_scan = InternetNLV2Scan()
new_scan.type = internal_scan_type
new_scan.save()
internet_nl_v2_websecmap.update_state(
new_scan.id, "requested and empty",
"requested a scan to be performed on internet.nl api")
# We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change:
accountinternetnlscan = AccountInternetNLScan()
accountinternetnlscan.account = urllist.account
accountinternetnlscan.urllist = urllist
accountinternetnlscan.started_on = datetime.now(pytz.utc)
accountinternetnlscan.scan = new_scan
accountinternetnlscan.state = ""
accountinternetnlscan.save()
# and start the process.
update_state("requested", accountinternetnlscan.id)
# Sprinkling an activity stream action.
action.send(urllist.account,
verb=f'started {manual_or_scheduled} scan',
target=accountinternetnlscan,
public=False)
return accountinternetnlscan.id
def initialize_scan(urllist: UrlList, manual_or_scheduled: str = "scheduled"):
# We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change:
translated_scan_types = {'web': 'web', 'mail': 'mail_dashboard'}
new_scan = InternetNLV2Scan()
new_scan.type = translated_scan_types[urllist.scan_type]
new_scan.save()
internet_nl_v2_websecmap.update_state(
new_scan, "requested and empty",
"requested a scan to be performed on internet.nl api")
accountinternetnlscan = AccountInternetNLScan()
accountinternetnlscan.account = urllist.account
accountinternetnlscan.urllist = urllist
accountinternetnlscan.started_on = datetime.now(pytz.utc)
accountinternetnlscan.scan = new_scan
accountinternetnlscan.state = ""
accountinternetnlscan.save()
# and start the process.
update_state("requested", accountinternetnlscan)
# Sprinkling an activity stream action.
action.send(urllist.account,
verb=f'started {manual_or_scheduled} scan',
target=accountinternetnlscan,
public=False)
return accountinternetnlscan
def create_scan_report(account: Account, urllist: UrlList):
rate_urllists_now([urllist])
# create a scan for this list,
scan, created = InternetNLV2Scan.objects.all().get_or_create(
pk=1, type='web', state="finished")
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = urllist
ainls.scan = scan
scan.finished = True
scan.finished_on = timezone.now()
ainls.save()
rate_urllists_now.si(urllist, prevent_duplicates=False)
def connect_scan_to_account(scan: InternetNLScan, account: Account,
urllist: UrlList) -> AccountInternetNLScan:
if not scan:
raise ValueError('Scan is empty')
if not account:
raise ValueError('Account is empty')
if not urllist:
raise ValueError('Urllist is empty')
scan_relation = AccountInternetNLScan()
scan_relation.account = account
scan_relation.scan = scan
scan_relation.urllist = urllist
scan_relation.save()
return scan_relation
def create_scan_report(account: Account, urllist: UrlList):
rate_urllists_now([urllist])
# create a scan for this list,
scan, created = InternetNLScan.objects.all().get_or_create(pk=1,
type='web',
success=False,
started=False,
finished=False)
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = urllist
ainls.scan = scan
scan.finished = True
scan.finished_on = timezone.now()
ainls.save()
# finished on is set, so we can make a report now...
create_reports_on_finished_scans(urllist)
def retroactively_add_scan(report, account, urllist):
scan = InternetNLScan()
# The scan is already perfomed and complete, but the values from the scan still have to be imported(!)
# therefore this is set to false...
scan.finished = False
scan.success = False
scan.status_url = f"{REPORT_BASE_URL}{report['report_id']}/"
scan.type = report['scan_type']
scan.finished_on = timezone.now()
scan.started_on = timezone.now()
scan.started = True
scan.message = "Imported manually"
scan.friendly_message = "Imported manually"
scan.last_check = timezone.now()
scan.save()
accountscan = AccountInternetNLScan()
accountscan.account = account
accountscan.urllist = urllist
accountscan.scan = scan
accountscan.save()
def test_rate_urllists(db, monkeypatch) -> None:
# mock get_allowed_to_report as constance tries to connect to redis.
# report everything and don't contact redis for constance values.
monkeypatch.setattr(websecmap.reporting.report, 'get_allowed_to_report', lambda: ALL_SCAN_TYPES)
account, url, endpoint, scan = make_url_with_endpoint_and_scan()
list = get_or_create_list_by_name(account, "test list 1")
_add_to_urls_to_urllist(account, list, [url])
# first rate the urls.
UrlReport.objects.all().delete()
create_url_report(create_timeline(url), url)
rate_urllists_now([list])
# We now should have 1 UrlListReport
assert UrlListReport.objects.all().count() == 1
#
#
#
#
# Now test on the creation of reports on finished scans.
#
#
#
#
UrlListReport.objects.all().delete()
# Then create a scan, in such a state that it cannot create a report:
# In this case there is no scan for this list at all. Should not result in any report.
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
# create a scan for this list,
scan, created = InternetNLScan.objects.all().get_or_create(
pk=1, type='web', success=False, started=False, finished=False)
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = list
ainls.scan = scan
ainls.save()
# scan has not finished yet, so no reports:
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
scan.finished = True
scan.save()
# While the scan has finished, the finished_on field is not set. This means that the results from the report have
# not yet been parsed.
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
scan.finished_on = timezone.now()
scan.save()
# finished on is set, so we can make a report now...
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 1