def create_scan(internal_scan_type: str, urllist: UrlList, manual_or_scheduled: str = "scheduled") -> int: new_scan = InternetNLV2Scan() new_scan.type = internal_scan_type new_scan.save() internet_nl_v2_websecmap.update_state( new_scan.id, "requested and empty", "requested a scan to be performed on internet.nl api") # We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change: accountinternetnlscan = AccountInternetNLScan() accountinternetnlscan.account = urllist.account accountinternetnlscan.urllist = urllist accountinternetnlscan.started_on = datetime.now(pytz.utc) accountinternetnlscan.scan = new_scan accountinternetnlscan.state = "" accountinternetnlscan.save() # and start the process. update_state("requested", accountinternetnlscan.id) # Sprinkling an activity stream action. action.send(urllist.account, verb=f'started {manual_or_scheduled} scan', target=accountinternetnlscan, public=False) return accountinternetnlscan.id
def initialize_scan(urllist: UrlList, manual_or_scheduled: str = "scheduled"): # We need to store the scan type in the InternetNLV2Scan at creation, because the type in the list might change: translated_scan_types = {'web': 'web', 'mail': 'mail_dashboard'} new_scan = InternetNLV2Scan() new_scan.type = translated_scan_types[urllist.scan_type] new_scan.save() internet_nl_v2_websecmap.update_state( new_scan, "requested and empty", "requested a scan to be performed on internet.nl api") accountinternetnlscan = AccountInternetNLScan() accountinternetnlscan.account = urllist.account accountinternetnlscan.urllist = urllist accountinternetnlscan.started_on = datetime.now(pytz.utc) accountinternetnlscan.scan = new_scan accountinternetnlscan.state = "" accountinternetnlscan.save() # and start the process. update_state("requested", accountinternetnlscan) # Sprinkling an activity stream action. action.send(urllist.account, verb=f'started {manual_or_scheduled} scan', target=accountinternetnlscan, public=False) return accountinternetnlscan
def setup_test(): user = User( **{ 'first_name': 'test', 'last_name': 'test', 'username': '******', 'is_active': True }) user.save() account = Account(**{'name': 'test'}) account.save() dashboarduser = DashboardUser( **{ 'mail_preferred_mail_address': '*****@*****.**', 'mail_preferred_language': 'nl', 'mail_send_mail_after_scan_finished': True, 'account': account, 'user': user }) dashboarduser.save() urllist = UrlList(**{'name': '', 'account': account}) urllist.save() urllistreport = UrlListReport( **{ 'urllist': urllist, 'average_internet_nl_score': 42.42, 'at_when': timezone.now() }) urllistreport.save() internetnlv2scan = InternetNLV2Scan(**{ 'type': 'web', 'scan_id': '123', 'state': 'finished' }) internetnlv2scan.save() accountinternetnlscan = AccountInternetNLScan( **{ 'account': account, 'scan': internetnlv2scan, 'urllist': urllist, 'started_on': timezone.now(), 'report': urllistreport, 'finished_on': timezone.now() + timedelta(hours=2) }) accountinternetnlscan.save() # first template: template = EmailTemplate() template.name = "scan_finished_en" template.subject = "test" template.description = "test" template.email_html_text = "test {{report_average_internet_nl_score}}." template.save()
def create_scan_report(account: Account, urllist: UrlList): rate_urllists_now([urllist]) # create a scan for this list, scan, created = InternetNLV2Scan.objects.all().get_or_create( pk=1, type='web', state="finished") ainls = AccountInternetNLScan() ainls.account = account ainls.urllist = urllist ainls.scan = scan scan.finished = True scan.finished_on = timezone.now() ainls.save() rate_urllists_now.si(urllist, prevent_duplicates=False)
def connect_scan_to_account(scan: InternetNLScan, account: Account, urllist: UrlList) -> AccountInternetNLScan: if not scan: raise ValueError('Scan is empty') if not account: raise ValueError('Account is empty') if not urllist: raise ValueError('Urllist is empty') scan_relation = AccountInternetNLScan() scan_relation.account = account scan_relation.scan = scan scan_relation.urllist = urllist scan_relation.save() return scan_relation
def create_scan_report(account: Account, urllist: UrlList): rate_urllists_now([urllist]) # create a scan for this list, scan, created = InternetNLScan.objects.all().get_or_create(pk=1, type='web', success=False, started=False, finished=False) ainls = AccountInternetNLScan() ainls.account = account ainls.urllist = urllist ainls.scan = scan scan.finished = True scan.finished_on = timezone.now() ainls.save() # finished on is set, so we can make a report now... create_reports_on_finished_scans(urllist)
def test_update_state(db): account = Account() account.save() urllist = UrlList(**{'name': '', 'account': account}) urllist.save() scan = AccountInternetNLScan() scan.urllist = urllist scan.account = account scan.save() update_state("new", scan.id) # A situation has occurred where the log was already in the next step, but the scan state itself was # at the old step. This caused the scan to block: as the logic was that if the log is in the correct / new state # it was assumed the scan was also in that state. This is fixed and tested below. scanlog = AccountInternetNLScanLog() scanlog.scan = scan scanlog.at_when = timezone.now() scanlog.state = "out of sync" scanlog.save() update_state("out of sync", scan.id) my_scan = AccountInternetNLScan.objects.all().first() assert my_scan.state == "out of sync" # new, out of sync, out of sync # The last duplicate is stored to make it clear something went wrong. There is nothing 'attached' to the log # other than understanding the process in case of weird situations. assert AccountInternetNLScanLog.objects.all().count() == 3 # make sure the amount of log info does not grow if things are the same update_state("out of sync", my_scan.id) update_state("out of sync", my_scan.id) update_state("out of sync", my_scan.id) assert AccountInternetNLScanLog.objects.all().count() == 3
def retroactively_add_scan(report, account, urllist): scan = InternetNLScan() # The scan is already perfomed and complete, but the values from the scan still have to be imported(!) # therefore this is set to false... scan.finished = False scan.success = False scan.status_url = f"{REPORT_BASE_URL}{report['report_id']}/" scan.type = report['scan_type'] scan.finished_on = timezone.now() scan.started_on = timezone.now() scan.started = True scan.message = "Imported manually" scan.friendly_message = "Imported manually" scan.last_check = timezone.now() scan.save() accountscan = AccountInternetNLScan() accountscan.account = account accountscan.urllist = urllist accountscan.scan = scan accountscan.save()
def test_report_upgrade(db, monkeypatch) -> None: # Create urllist with a lot of unscannable domains, only apple.com is scannable. # megaupload.com will never be scannable, and the rest can have an endpoint and might be in the report # already because of this (but without endpoints) urls = ['akamaihd.net', 'apple.com', 'bp.blogspot.com', 'clickbank.net', 'cocolog-nifty.com', 'fda.gov', 'geocities.jp', 'ggpht.com', 'googleusercontent.com', 'megaupload.com', 'nhk.or.jp', 'ssl-images-amazon.com', 'ytimg.com'] # create the list, code from test domain management: account, created = Account.objects.all().get_or_create(name="test") urllist = UrlList() urllist.name = "upgrade" urllist.account = account urllist.save() scan = AccountInternetNLScan() scan.urllist = urllist scan.account = account scan.save() for url in urls: new_url = Url() new_url.url = url new_url.save() urllist.urls.add(new_url) urllist.save() # fake a report on these domains, without any upgrades, taken from the acc environment: fake_calculation = { "high": 19, "medium": 4, "low": 3, "ok": 15, "total_urls": 1, "high_urls": 1, "medium_urls": 0, "low_urls": 0, "ok_urls": 0, "explained_high": 0, "explained_medium": 0, "explained_low": 0, "explained_high_endpoints": 0, "explained_medium_endpoints": 0, "explained_low_endpoints": 0, "explained_high_urls": 0, "explained_medium_urls": 0, "explained_low_urls": 0, "explained_total_url_issues": 0, "explained_url_issues_high": 0, "explained_url_issues_medium": 0, "explained_url_issues_low": 0, "explained_total_endpoint_issues": 0, "explained_endpoint_issues_high": 0, "explained_endpoint_issues_medium": 0, "explained_endpoint_issues_low": 0, "total_endpoints": 1, "high_endpoints": 1, "medium_endpoints": 0, "low_endpoints": 0, "ok_endpoints": 0, "total_url_issues": 0, "total_endpoint_issues": 26, "url_issues_high": 0, "url_issues_medium": 0, "url_issues_low": 0, "endpoint_issues_high": 19, "endpoint_issues_medium": 4, "endpoint_issues_low": 3, "urls": [ { "url": "apple.com", "ratings": [], "endpoints": [ { "id": 4599, "concat": "dns_a_aaaa/0 IPv0", "ip": 0, "ip_version": 0, "port": 0, "protocol": "dns_a_aaaa", "v4": False, "ratings": [ { "type": "internet_nl_web_ipv6_ws_address", "explanation": "Test internet_nl_web_ipv6_ws_address resulted in failed.", "since": "2020-01-15T13:00:01.116013+00:00", "last_scan": "2020-01-15T13:00:01.116689+00:00", "high": 1, "medium": 0, "low": 0, "ok": 0, "not_testable": False, "not_applicable": False, "error_in_test": False, "is_explained": False, "comply_or_explain_explanation": "", "comply_or_explain_explained_on": "", "comply_or_explain_explanation_valid_until": "", "comply_or_explain_valid_at_time_of_report": False, "scan": 114575, "scan_type": "internet_nl_web_ipv6_ws_address" }, { "type": "internet_nl_web_dnssec_valid", "explanation": "Test internet_nl_web_dnssec_valid resulted in failed.", "since": "2020-01-15T13:00:00.684906+00:00", "last_scan": "2020-01-15T13:00:00.685193+00:00", "high": 1, "medium": 0, "low": 0, "ok": 0, "not_testable": False, "not_applicable": False, "error_in_test": False, "is_explained": False, "comply_or_explain_explanation": "", "comply_or_explain_explained_on": "", "comply_or_explain_explanation_valid_until": "", "comply_or_explain_valid_at_time_of_report": False, "scan": 114556, "scan_type": "internet_nl_web_dnssec_valid" }, ], "high": 19, "medium": 4, "low": 3, "ok": 15, "explained_high": 0, "explained_medium": 0, "explained_low": 0 } ], "total_issues": 26, "high": 19, "medium": 4, "low": 3, "ok": 15, "total_endpoints": 1, "high_endpoints": 1, "medium_endpoints": 0, "low_endpoints": 0, "ok_endpoints": 0, "total_url_issues": 0, "url_issues_high": 0, "url_issues_medium": 0, "url_issues_low": 0, "url_ok": 0, "total_endpoint_issues": 26, "endpoint_issues_high": 19, "endpoint_issues_medium": 4, "endpoint_issues_low": 3, "explained_total_issues": 0, "explained_high": 0, "explained_medium": 0, "explained_low": 0, "explained_high_endpoints": 0, "explained_medium_endpoints": 0, "explained_low_endpoints": 0, "explained_total_url_issues": 0, "explained_url_issues_high": 0, "explained_url_issues_medium": 0, "explained_url_issues_low": 0, "explained_total_endpoint_issues": 0, "explained_endpoint_issues_high": 0, "explained_endpoint_issues_medium": 0, "explained_endpoint_issues_low": 0 } ], "total_issues": 26, "name": "Unscannable Web + one scannable" } fake_report = UrlListReport() fake_report.calculation = fake_calculation fake_report.urllist = urllist fake_report.at_when = timezone.now() fake_report.save() # First check if we are removing the comply_or_explain keys, mainly to save data: remove_comply_or_explain(fake_calculation) assert "explained_endpoint_issues_high" not in fake_calculation['urls'][0] assert "comply_or_explain_explanation" not in fake_calculation['urls'][0]['endpoints'][0]["ratings"][0] # Now add ratings based on keys, which makes direct access possible: add_keyed_ratings(fake_calculation) assert "ratings_by_type" in fake_calculation['urls'][0]['endpoints'][0] assert "internet_nl_web_ipv6_ws_address" in fake_calculation['urls'][0]['endpoints'][0]['ratings_by_type'] # Add graph statistics, so the graphs can be instantly created based on report data add_statistics_over_ratings(fake_calculation) assert "statistics_per_issue_type" in fake_calculation assert "internet_nl_web_ipv6_ws_address" in fake_calculation["statistics_per_issue_type"] # todo: we can add some tests here to see if the aggregation is correct # add some statistics over all these metrics add_percentages_to_statistics(fake_calculation) assert "pct_ok" in fake_calculation["statistics_per_issue_type"]["internet_nl_web_ipv6_ws_address"] # and make sure the report is complete: meaning that all urls requested are present, even though they # could not be scanned. So a top 100 stays a top 100. assert (len(fake_calculation['urls']) == 1) upgrade_report_with_unscannable_urls(fake_report.id, scan.id) fake_report = UrlListReport.objects.all().first() assert(len(fake_report.calculation['urls']) == len(urls)) # the first url should still be by apple: assert fake_report.calculation['urls'][0]['url'] == "apple.com"
def test_rate_urllists(db, monkeypatch) -> None: # mock get_allowed_to_report as constance tries to connect to redis. # report everything and don't contact redis for constance values. monkeypatch.setattr(websecmap.reporting.report, 'get_allowed_to_report', lambda: ALL_SCAN_TYPES) account, url, endpoint, scan = make_url_with_endpoint_and_scan() list = get_or_create_list_by_name(account, "test list 1") _add_to_urls_to_urllist(account, list, [url]) # first rate the urls. UrlReport.objects.all().delete() create_url_report(create_timeline(url), url) rate_urllists_now([list]) # We now should have 1 UrlListReport assert UrlListReport.objects.all().count() == 1 # # # # # Now test on the creation of reports on finished scans. # # # # UrlListReport.objects.all().delete() # Then create a scan, in such a state that it cannot create a report: # In this case there is no scan for this list at all. Should not result in any report. create_reports_on_finished_scans(list) assert UrlListReport.objects.all().count() == 0 # create a scan for this list, scan, created = InternetNLScan.objects.all().get_or_create( pk=1, type='web', success=False, started=False, finished=False) ainls = AccountInternetNLScan() ainls.account = account ainls.urllist = list ainls.scan = scan ainls.save() # scan has not finished yet, so no reports: create_reports_on_finished_scans(list) assert UrlListReport.objects.all().count() == 0 scan.finished = True scan.save() # While the scan has finished, the finished_on field is not set. This means that the results from the report have # not yet been parsed. create_reports_on_finished_scans(list) assert UrlListReport.objects.all().count() == 0 scan.finished_on = timezone.now() scan.save() # finished on is set, so we can make a report now... create_reports_on_finished_scans(list) assert UrlListReport.objects.all().count() == 1