def update_state(state: str, scan: AccountInternetNLScan) -> None:
"""Update the current scan state. Also write it to the scan log. From this log we should also be able to see
retries... when celery retries on exceptions etc..."""
# if the state is still the same, just update the last_check, don't append the log.
# Don't get it from the scan object, that info might be obsolete.
last_state_for_scan = AccountInternetNLScanLog.objects.all().filter(
scan=scan).order_by("-at_when").first()
if last_state_for_scan:
if last_state_for_scan.state == state:
return
# do not update a cancelled scan (#159), even if a certain task has finished after a cancel was issued (letting the
# task overwriting the cancelled state, continuing the scan)
if last_state_for_scan == "cancelled":
return
# First state, or a new state.
# New log message:
scan.state = state
scan.state_changed_on = timezone.now()
scan.save()
scanlog = AccountInternetNLScanLog()
scanlog.scan = scan
scanlog.at_when = timezone.now()
scanlog.state = state
scanlog.save()
def finishing_scan(scan: AccountInternetNLScan):
# No further actions, so not setting "finishing scan" as a state, but set it to "scan finished" directly.
scan.finished_on = datetime.now(pytz.utc)
scan.save()
update_state("finished", scan)
return group([])
def setup_test():
user = User(
**{
'first_name': 'test',
'last_name': 'test',
'username': '******',
'is_active': True
})
user.save()
account = Account(**{'name': 'test'})
account.save()
dashboarduser = DashboardUser(
**{
'mail_preferred_mail_address': '*****@*****.**',
'mail_preferred_language': 'nl',
'mail_send_mail_after_scan_finished': True,
'account': account,
'user': user
})
dashboarduser.save()
urllist = UrlList(**{'name': '', 'account': account})
urllist.save()
urllistreport = UrlListReport(
**{
'urllist': urllist,
'average_internet_nl_score': 42.42,
'at_when': timezone.now()
})
urllistreport.save()
internetnlv2scan = InternetNLV2Scan(**{
'type': 'web',
'scan_id': '123',
'state': 'finished'
})
internetnlv2scan.save()
accountinternetnlscan = AccountInternetNLScan(
**{
'account': account,
'scan': internetnlv2scan,
'urllist': urllist,
'started_on': timezone.now(),
'report': urllistreport,
'finished_on': timezone.now() + timedelta(hours=2)
})
accountinternetnlscan.save()
# first template:
template = EmailTemplate()
template.name = "scan_finished_en"
template.subject = "test"
template.description = "test"
template.email_html_text = "test {{report_average_internet_nl_score}}."
template.save()
def initialize_scan(urllist: UrlList, manual_or_scheduled: str = "scheduled"):
# We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change:
translated_scan_types = {'web': 'web', 'mail': 'mail_dashboard'}
new_scan = InternetNLV2Scan()
new_scan.type = translated_scan_types[urllist.scan_type]
new_scan.save()
internet_nl_v2_websecmap.update_state(
new_scan, "requested and empty",
"requested a scan to be performed on internet.nl api")
accountinternetnlscan = AccountInternetNLScan()
accountinternetnlscan.account = urllist.account
accountinternetnlscan.urllist = urllist
accountinternetnlscan.started_on = datetime.now(pytz.utc)
accountinternetnlscan.scan = new_scan
accountinternetnlscan.state = ""
accountinternetnlscan.save()
# and start the process.
update_state("requested", accountinternetnlscan)
# Sprinkling an activity stream action.
action.send(urllist.account,
verb=f'started {manual_or_scheduled} scan',
target=accountinternetnlscan,
public=False)
return accountinternetnlscan
def create_scan(internal_scan_type: str,
urllist: UrlList,
manual_or_scheduled: str = "scheduled") -> int:
new_scan = InternetNLV2Scan()
new_scan.type = internal_scan_type
new_scan.save()
internet_nl_v2_websecmap.update_state(
new_scan.id, "requested and empty",
"requested a scan to be performed on internet.nl api")
# We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change:
accountinternetnlscan = AccountInternetNLScan()
accountinternetnlscan.account = urllist.account
accountinternetnlscan.urllist = urllist
accountinternetnlscan.started_on = datetime.now(pytz.utc)
accountinternetnlscan.scan = new_scan
accountinternetnlscan.state = ""
accountinternetnlscan.save()
# and start the process.
update_state("requested", accountinternetnlscan.id)
# Sprinkling an activity stream action.
action.send(urllist.account,
verb=f'started {manual_or_scheduled} scan',
target=accountinternetnlscan,
public=False)
return accountinternetnlscan.id
def create_scan_report(account: Account, urllist: UrlList):
rate_urllists_now([urllist])
# create a scan for this list,
scan, created = InternetNLV2Scan.objects.all().get_or_create(
pk=1, type='web', state="finished")
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = urllist
ainls.scan = scan
scan.finished = True
scan.finished_on = timezone.now()
ainls.save()
rate_urllists_now.si(urllist, prevent_duplicates=False)
def connect_scan_to_account(scan: InternetNLScan, account: Account,
urllist: UrlList) -> AccountInternetNLScan:
if not scan:
raise ValueError('Scan is empty')
if not account:
raise ValueError('Account is empty')
if not urllist:
raise ValueError('Urllist is empty')
scan_relation = AccountInternetNLScan()
scan_relation.account = account
scan_relation.scan = scan
scan_relation.urllist = urllist
scan_relation.save()
return scan_relation
def create_scan_report(account: Account, urllist: UrlList):
rate_urllists_now([urllist])
# create a scan for this list,
scan, created = InternetNLScan.objects.all().get_or_create(pk=1,
type='web',
success=False,
started=False,
finished=False)
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = urllist
ainls.scan = scan
scan.finished = True
scan.finished_on = timezone.now()
ainls.save()
# finished on is set, so we can make a report now...
create_reports_on_finished_scans(urllist)
def retroactively_add_scan(report, account, urllist):
scan = InternetNLScan()
# The scan is already perfomed and complete, but the values from the scan still have to be imported(!)
# therefore this is set to false...
scan.finished = False
scan.success = False
scan.status_url = f"{REPORT_BASE_URL}{report['report_id']}/"
scan.type = report['scan_type']
scan.finished_on = timezone.now()
scan.started_on = timezone.now()
scan.started = True
scan.message = "Imported manually"
scan.friendly_message = "Imported manually"
scan.last_check = timezone.now()
scan.save()
accountscan = AccountInternetNLScan()
accountscan.account = account
accountscan.urllist = urllist
accountscan.scan = scan
accountscan.save()
def test_update_state(db):
account = Account()
account.save()
urllist = UrlList(**{'name': '', 'account': account})
urllist.save()
scan = AccountInternetNLScan()
scan.urllist = urllist
scan.account = account
scan.save()
update_state("new", scan.id)
# A situation has occurred where the log was already in the next step, but the scan state itself was
# at the old step. This caused the scan to block: as the logic was that if the log is in the correct / new state
# it was assumed the scan was also in that state. This is fixed and tested below.
scanlog = AccountInternetNLScanLog()
scanlog.scan = scan
scanlog.at_when = timezone.now()
scanlog.state = "out of sync"
scanlog.save()
update_state("out of sync", scan.id)
my_scan = AccountInternetNLScan.objects.all().first()
assert my_scan.state == "out of sync"
# new, out of sync, out of sync
# The last duplicate is stored to make it clear something went wrong. There is nothing 'attached' to the log
# other than understanding the process in case of weird situations.
assert AccountInternetNLScanLog.objects.all().count() == 3
# make sure the amount of log info does not grow if things are the same
update_state("out of sync", my_scan.id)
update_state("out of sync", my_scan.id)
update_state("out of sync", my_scan.id)
assert AccountInternetNLScanLog.objects.all().count() == 3
def connect_urllistreport_to_accountinternetnlscan(
urllistreport: UrlListReport, scan: AccountInternetNLScan):
scan.report = urllistreport
scan.save()
return urllistreport
def test_report_upgrade(db, monkeypatch) -> None:
# Create urllist with a lot of unscannable domains, only apple.com is scannable.
# megaupload.com will never be scannable, and the rest can have an endpoint and might be in the report
# already because of this (but without endpoints)
urls = ['akamaihd.net', 'apple.com', 'bp.blogspot.com', 'clickbank.net', 'cocolog-nifty.com', 'fda.gov',
'geocities.jp', 'ggpht.com', 'googleusercontent.com', 'megaupload.com', 'nhk.or.jp',
'ssl-images-amazon.com', 'ytimg.com']
# create the list, code from test domain management:
account, created = Account.objects.all().get_or_create(name="test")
urllist = UrlList()
urllist.name = "upgrade"
urllist.account = account
urllist.save()
scan = AccountInternetNLScan()
scan.urllist = urllist
scan.account = account
scan.save()
for url in urls:
new_url = Url()
new_url.url = url
new_url.save()
urllist.urls.add(new_url)
urllist.save()
# fake a report on these domains, without any upgrades, taken from the acc environment:
fake_calculation = {
"high": 19,
"medium": 4,
"low": 3,
"ok": 15,
"total_urls": 1,
"high_urls": 1,
"medium_urls": 0,
"low_urls": 0,
"ok_urls": 0,
"explained_high": 0,
"explained_medium": 0,
"explained_low": 0,
"explained_high_endpoints": 0,
"explained_medium_endpoints": 0,
"explained_low_endpoints": 0,
"explained_high_urls": 0,
"explained_medium_urls": 0,
"explained_low_urls": 0,
"explained_total_url_issues": 0,
"explained_url_issues_high": 0,
"explained_url_issues_medium": 0,
"explained_url_issues_low": 0,
"explained_total_endpoint_issues": 0,
"explained_endpoint_issues_high": 0,
"explained_endpoint_issues_medium": 0,
"explained_endpoint_issues_low": 0,
"total_endpoints": 1,
"high_endpoints": 1,
"medium_endpoints": 0,
"low_endpoints": 0,
"ok_endpoints": 0,
"total_url_issues": 0,
"total_endpoint_issues": 26,
"url_issues_high": 0,
"url_issues_medium": 0,
"url_issues_low": 0,
"endpoint_issues_high": 19,
"endpoint_issues_medium": 4,
"endpoint_issues_low": 3,
"urls": [
{
"url": "apple.com",
"ratings": [],
"endpoints": [
{
"id": 4599,
"concat": "dns_a_aaaa/0 IPv0",
"ip": 0,
"ip_version": 0,
"port": 0,
"protocol": "dns_a_aaaa",
"v4": False,
"ratings": [
{
"type": "internet_nl_web_ipv6_ws_address",
"explanation": "Test internet_nl_web_ipv6_ws_address resulted in failed.",
"since": "2020-01-15T13:00:01.116013+00:00",
"last_scan": "2020-01-15T13:00:01.116689+00:00",
"high": 1,
"medium": 0,
"low": 0,
"ok": 0,
"not_testable": False,
"not_applicable": False,
"error_in_test": False,
"is_explained": False,
"comply_or_explain_explanation": "",
"comply_or_explain_explained_on": "",
"comply_or_explain_explanation_valid_until": "",
"comply_or_explain_valid_at_time_of_report": False,
"scan": 114575,
"scan_type": "internet_nl_web_ipv6_ws_address"
},
{
"type": "internet_nl_web_dnssec_valid",
"explanation": "Test internet_nl_web_dnssec_valid resulted in failed.",
"since": "2020-01-15T13:00:00.684906+00:00",
"last_scan": "2020-01-15T13:00:00.685193+00:00",
"high": 1,
"medium": 0,
"low": 0,
"ok": 0,
"not_testable": False,
"not_applicable": False,
"error_in_test": False,
"is_explained": False,
"comply_or_explain_explanation": "",
"comply_or_explain_explained_on": "",
"comply_or_explain_explanation_valid_until": "",
"comply_or_explain_valid_at_time_of_report": False,
"scan": 114556,
"scan_type": "internet_nl_web_dnssec_valid"
},
],
"high": 19,
"medium": 4,
"low": 3,
"ok": 15,
"explained_high": 0,
"explained_medium": 0,
"explained_low": 0
}
],
"total_issues": 26,
"high": 19,
"medium": 4,
"low": 3,
"ok": 15,
"total_endpoints": 1,
"high_endpoints": 1,
"medium_endpoints": 0,
"low_endpoints": 0,
"ok_endpoints": 0,
"total_url_issues": 0,
"url_issues_high": 0,
"url_issues_medium": 0,
"url_issues_low": 0,
"url_ok": 0,
"total_endpoint_issues": 26,
"endpoint_issues_high": 19,
"endpoint_issues_medium": 4,
"endpoint_issues_low": 3,
"explained_total_issues": 0,
"explained_high": 0,
"explained_medium": 0,
"explained_low": 0,
"explained_high_endpoints": 0,
"explained_medium_endpoints": 0,
"explained_low_endpoints": 0,
"explained_total_url_issues": 0,
"explained_url_issues_high": 0,
"explained_url_issues_medium": 0,
"explained_url_issues_low": 0,
"explained_total_endpoint_issues": 0,
"explained_endpoint_issues_high": 0,
"explained_endpoint_issues_medium": 0,
"explained_endpoint_issues_low": 0
}
],
"total_issues": 26,
"name": "Unscannable Web + one scannable"
}
fake_report = UrlListReport()
fake_report.calculation = fake_calculation
fake_report.urllist = urllist
fake_report.at_when = timezone.now()
fake_report.save()
# First check if we are removing the comply_or_explain keys, mainly to save data:
remove_comply_or_explain(fake_calculation)
assert "explained_endpoint_issues_high" not in fake_calculation['urls'][0]
assert "comply_or_explain_explanation" not in fake_calculation['urls'][0]['endpoints'][0]["ratings"][0]
# Now add ratings based on keys, which makes direct access possible:
add_keyed_ratings(fake_calculation)
assert "ratings_by_type" in fake_calculation['urls'][0]['endpoints'][0]
assert "internet_nl_web_ipv6_ws_address" in fake_calculation['urls'][0]['endpoints'][0]['ratings_by_type']
# Add graph statistics, so the graphs can be instantly created based on report data
add_statistics_over_ratings(fake_calculation)
assert "statistics_per_issue_type" in fake_calculation
assert "internet_nl_web_ipv6_ws_address" in fake_calculation["statistics_per_issue_type"]
# todo: we can add some tests here to see if the aggregation is correct
# add some statistics over all these metrics
add_percentages_to_statistics(fake_calculation)
assert "pct_ok" in fake_calculation["statistics_per_issue_type"]["internet_nl_web_ipv6_ws_address"]
# and make sure the report is complete: meaning that all urls requested are present, even though they
# could not be scanned. So a top 100 stays a top 100.
assert (len(fake_calculation['urls']) == 1)
upgrade_report_with_unscannable_urls(fake_report.id, scan.id)
fake_report = UrlListReport.objects.all().first()
assert(len(fake_report.calculation['urls']) == len(urls))
# the first url should still be by apple:
assert fake_report.calculation['urls'][0]['url'] == "apple.com"
def test_rate_urllists(db, monkeypatch) -> None:
# mock get_allowed_to_report as constance tries to connect to redis.
# report everything and don't contact redis for constance values.
monkeypatch.setattr(websecmap.reporting.report, 'get_allowed_to_report', lambda: ALL_SCAN_TYPES)
account, url, endpoint, scan = make_url_with_endpoint_and_scan()
list = get_or_create_list_by_name(account, "test list 1")
_add_to_urls_to_urllist(account, list, [url])
# first rate the urls.
UrlReport.objects.all().delete()
create_url_report(create_timeline(url), url)
rate_urllists_now([list])
# We now should have 1 UrlListReport
assert UrlListReport.objects.all().count() == 1
#
#
#
#
# Now test on the creation of reports on finished scans.
#
#
#
#
UrlListReport.objects.all().delete()
# Then create a scan, in such a state that it cannot create a report:
# In this case there is no scan for this list at all. Should not result in any report.
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
# create a scan for this list,
scan, created = InternetNLScan.objects.all().get_or_create(
pk=1, type='web', success=False, started=False, finished=False)
ainls = AccountInternetNLScan()
ainls.account = account
ainls.urllist = list
ainls.scan = scan
ainls.save()
# scan has not finished yet, so no reports:
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
scan.finished = True
scan.save()
# While the scan has finished, the finished_on field is not set. This means that the results from the report have
# not yet been parsed.
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 0
scan.finished_on = timezone.now()
scan.save()
# finished on is set, so we can make a report now...
create_reports_on_finished_scans(list)
assert UrlListReport.objects.all().count() == 1