def setup_test(): user = User( **{ 'first_name': 'test', 'last_name': 'test', 'username': '******', 'is_active': True }) user.save() account = Account(**{'name': 'test'}) account.save() dashboarduser = DashboardUser( **{ 'mail_preferred_mail_address': '*****@*****.**', 'mail_preferred_language': 'nl', 'mail_send_mail_after_scan_finished': True, 'account': account, 'user': user }) dashboarduser.save() urllist = UrlList(**{'name': '', 'account': account}) urllist.save() urllistreport = UrlListReport( **{ 'urllist': urllist, 'average_internet_nl_score': 42.42, 'at_when': timezone.now() }) urllistreport.save() internetnlv2scan = InternetNLV2Scan(**{ 'type': 'web', 'scan_id': '123', 'state': 'finished' }) internetnlv2scan.save() accountinternetnlscan = AccountInternetNLScan( **{ 'account': account, 'scan': internetnlv2scan, 'urllist': urllist, 'started_on': timezone.now(), 'report': urllistreport, 'finished_on': timezone.now() + timedelta(hours=2) }) accountinternetnlscan.save() # first template: template = EmailTemplate() template.name = "scan_finished_en" template.subject = "test" template.description = "test" template.email_html_text = "test {{report_average_internet_nl_score}}." template.save()
def test_urllistreport_get_previous_report(db): account = Account(**{'name': 'test'}) account.save() u = UrlList(**{'name': '', 'account': account}) u.save() urllistreport1 = UrlListReport( **{ 'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 1) }) urllistreport1.save() urllistreport2 = UrlListReport( **{ 'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 2) }) urllistreport2.save() urllistreport3 = UrlListReport( **{ 'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 3) }) urllistreport3.save() urllistreport4 = UrlListReport( **{ 'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 4) }) urllistreport4.save() urllistreport5 = UrlListReport( **{ 'urllist': u, 'average_internet_nl_score': 1, 'at_when': datetime(2020, 5, 5) }) urllistreport5.save() assert urllistreport5.get_previous_report_from_this_list( ) == urllistreport4 assert urllistreport4.get_previous_report_from_this_list( ) == urllistreport3 assert urllistreport3.get_previous_report_from_this_list( ) == urllistreport2 assert urllistreport2.get_previous_report_from_this_list( ) == urllistreport1 assert urllistreport1.get_previous_report_from_this_list() is None
def test_password_storage(db) -> None: # normal usage secret_password = '******' account, created = Account.objects.all().get_or_create(name="test") account.internet_nl_api_password = account.encrypt_password(secret_password) account.save() # this does not perform a retrieval, so of course it's the same. assert account.decrypt_password() == secret_password # no password set, should throw a valueError with pytest.raises(ValueError, match=r'.*not set.*'): account, created = Account.objects.all().get_or_create(name="value error") account.decrypt_password() # can create at once: account2 = Account() account2.name = 'test 2' account2.internet_nl_api_username = '******' account2.internet_nl_api_password = Account.encrypt_password(secret_password) account2.save() assert account2.decrypt_password() == secret_password # If the field is stored as CharField instead of BinaryField, the type has become a string. # It's a bit hard to cast that string back into bytes. # verify that when retrieving all accounts, the password fields are still bytes. accounts = Account.objects.all().filter(name__in=['test', 'test 2']) for account in accounts: # assert type(account.internet_nl_api_password) is bytes assert account.decrypt_password() == secret_password
def create_dashboard_scan_task(account: Account, urllist: UrlList, save_as_scan_type: str, endpoint_type: str) -> Task: # The scan name is arbitrary. Add a lot of info to it so the scan can be tracked. # A UUID will be added during registering scan_name = "{'source': 'Internet.nl Dashboard', 'type': '%s', 'account': '%s', 'list': '%s'}" % ( save_as_scan_type, account.name, urllist.name) api_url = API_URL_WEB if save_as_scan_type == 'web' else API_URL_MAIL return ( # Should we only try to get the specifically needed dns_endpoint? At what volume we should / must? # This discovers dns_endpoints. On the basis of this we know what urls we should scan an which # ones we should not. We'll only scan if there are valid endpoint, just like at internet.nl compose_discover_task( **{ 'urls_filter': { 'urls_in_dashboard_list': urllist, 'is_dead': False, 'not_resolvable': False } }) # Make sure that the discovery as listed above is actually used in the scan | get_relevant_urls.si(urllist, endpoint_type) # The urls are registered as part of the scan | register_scan.s(account.internet_nl_api_username, account.decrypt_password(), save_as_scan_type, api_url, scan_name) # When the scan is created, the scan is connected to the account for tracking purposes. # This is visualized in the scan monitor. | connect_scan_to_account.s(account, urllist))
def test_per_acount_scanner(db) -> None: account, created = Account.objects.all().get_or_create( name="test", internet_nl_api_username='******', internet_nl_api_password=Account.encrypt_password('test'), ) # some test urls url1, created = Url.objects.all().get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False) url2, created = Url.objects.all().get_or_create(url='internet.nl', is_dead=False, not_resolvable=False) url3, created = Url.objects.all().get_or_create(url='nu.nl', is_dead=False, not_resolvable=False) urls = [url1, url2, url3] # a set of standard endpoints ep = Endpoint( **{ 'url': urls[0], 'protocol': 'dns_a_aaaa', 'port': 0, 'ip_version': 0, 'is_dead': False }) ep.save() ep = Endpoint( **{ 'url': urls[1], 'protocol': 'dns_soa', 'port': 0, 'ip_version': 0, 'is_dead': False }) ep.save() # create an url list for this account: urllist, created = UrlList.objects.all().get_or_create(name='test', account=account, scan_type='web') for url in urls: urllist.urls.add(url) urllist.save() # When filtering for an existing account, as created above, there should be two tasks created. filters = {'account_filters': {'name': 'test'}} scan_tasks = compose_task(**filters) # two tasks: one for mail_dashboard, one for web, no type delivers one task but no content (should improve test). assert len(scan_tasks.kwargs.get('tasks')) == 1 # This should not result in any scans, as the account does not exist, an empty group is returned. filters = {'account_filters': {'name': 'not existing'}} scan_tasks = compose_task(**filters) log.debug(scan_tasks) assert scan_tasks.kwargs.get('tasks') == []
def save_model(self, request, obj, form, change): # If the internet_nl_api_password changed, encrypt the new value. # Example usage and docs: https://github.com/pyca/cryptography if 'new_password' in form.changed_data: obj.internet_nl_api_password = Account.encrypt_password(form.cleaned_data.get('new_password')) # check if the username / password combination is valid super().save_model(request, obj, form, change)
def test_password_storage(db) -> None: # normal usage secret_password = '******' account, created = Account.objects.all().get_or_create(name="test") account.internet_nl_api_password = account.encrypt_password( secret_password) account.save() # this does not perform a retrieval, so of course it's the same. assert account.decrypt_password() == secret_password # no password set, should throw a valueError with pytest.raises(ValueError, match=r'.*not set.*'): account, created = Account.objects.all().get_or_create( name="value error") account.decrypt_password() with transaction.atomic(): # there is no magic going on. Passwords have to be encrypted manually. # Warning: the transaction is broken here. # An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block. with pytest.raises(TypeError, match=r'.*bytes-like.*'): account.internet_nl_api_password = "******" account.save() with pytest.raises(ValueError, match=r'.*not encrypted.*'): account.decrypt_password() # can create at once: account2 = Account() account2.name = 'test 2' account2.internet_nl_api_username = '******' account2.internet_nl_api_password = Account.encrypt_password( secret_password) account2.save() assert account2.decrypt_password() == secret_password # If the field is stored as CharField instead of BinaryField, the type has become a string. # It's a bit hard to cast that string back into bytes. # verify that when retrieving all accounts, the password fields are still bytes. accounts = Account.objects.all().filter(name__in=['test', 'test 2']) for account in accounts: # assert type(account.internet_nl_api_password) is bytes assert account.decrypt_password() == secret_password
def test_update_state(db): account = Account() account.save() urllist = UrlList(**{'name': '', 'account': account}) urllist.save() scan = AccountInternetNLScan() scan.urllist = urllist scan.account = account scan.save() update_state("new", scan.id) # A situation has occurred where the log was already in the next step, but the scan state itself was # at the old step. This caused the scan to block: as the logic was that if the log is in the correct / new state # it was assumed the scan was also in that state. This is fixed and tested below. scanlog = AccountInternetNLScanLog() scanlog.scan = scan scanlog.at_when = timezone.now() scanlog.state = "out of sync" scanlog.save() update_state("out of sync", scan.id) my_scan = AccountInternetNLScan.objects.all().first() assert my_scan.state == "out of sync" # new, out of sync, out of sync # The last duplicate is stored to make it clear something went wrong. There is nothing 'attached' to the log # other than understanding the process in case of weird situations. assert AccountInternetNLScanLog.objects.all().count() == 3 # make sure the amount of log info does not grow if things are the same update_state("out of sync", my_scan.id) update_state("out of sync", my_scan.id) update_state("out of sync", my_scan.id) assert AccountInternetNLScanLog.objects.all().count() == 3
def save(self): cleaned_data = super().clean() username = cleaned_data.get("username") password = cleaned_data.get("password") user = User(**{'username': username}) user.set_password(password) user.is_active = True user.save() account = Account( **{ 'name': username, 'internet_nl_api_username': username, 'internet_nl_api_password': Account.encrypt_password(password), 'can_connect_to_internet_nl_api': Account.connect_to_internet_nl_api(username, password) }) account.save() dashboarduser = DashboardUser(**{'user': user, 'account': account}) dashboarduser.save()
def save_instant_account(request) -> HttpResponse: request = get_json_body(request) username = request['username'] password = request['password'] if User.objects.all().filter(username=username).exists(): return JsonResponse(operation_response(error=True, message=f"User with username '{username}' already exists.")) if Account.objects.all().filter(name=username).exists(): return JsonResponse(operation_response(error=True, message=f"Account with username {username}' already exists.")) # Extremely arbitrary password requirements. Just to make sure a password has been filled in. if len(password) < 5: return JsonResponse(operation_response(error=True, message=f"Password not filled in or not long enough.")) # all seems fine, let's add the user user = User(**{'username': username}) user.set_password(password) user.is_active = True user.save() account = Account(**{ 'name': username, 'internet_nl_api_username': username, 'internet_nl_api_password': Account.encrypt_password(password), 'can_connect_to_internet_nl_api': Account.connect_to_internet_nl_api(username, password) }) account.save() dashboarduser = DashboardUser(**{'user': user, 'account': account}) dashboarduser.save() return JsonResponse(operation_response(success=True, message=f"Account and user with name '{username}' created!"))
def test_retroactive_import(db, monkeypatch, redis_server): monkeypatch.setattr(requests, 'get', mocked_requests_get) account = Account() account.id = 2 account.name = "test" account.internet_nl_api_username = "******" account.internet_nl_api_password = account.encrypt_password("test") account.save() retroactively_import({ 'account_id': account.id, 'report_id': 'e738da28c0724e188dc808580fdcdf0e', 'scan_type': 'web' }) assert Url.objects.all().count() == 18 assert Endpoint.objects.all().count() == 18 assert InternetNLScan.objects.all().count() == 1 assert AccountInternetNLScan.objects.all().count() == 1
def test_per_acount_scanner(db) -> None: account, created = Account.objects.all().get_or_create(name="test", internet_nl_api_username='******', internet_nl_api_password=Account.encrypt_password('test'),) # add duplicate (named) urls, the name is the same, the id is different. url1, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, internal_notes='different data, is different id 1') url2, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, internal_notes='different data, is different id 2') url3, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False, internal_notes='different data, is different id 3') url4, created = Url.objects.get_or_create(url='www.unaffected_url.nl', is_dead=False, not_resolvable=False, internal_notes='different data, is different id 3') # make sure that these three urls actually exist assert Url.objects.all().count() == 4 # with duplicate endpoints, where the endpoints themselves are unique for the url. ep1, created = Endpoint.objects.get_or_create(url=url1, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) ep2, created = Endpoint.objects.get_or_create(url=url2, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) ep3, created = Endpoint.objects.get_or_create(url=url3, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False) ep4, created = Endpoint.objects.get_or_create(url=url1, protocol='dns_soa', port=0, ip_version=0, is_dead=False) ep5, created = Endpoint.objects.get_or_create(url=url2, protocol='dns_soa', port=0, ip_version=0, is_dead=False) ep6, created = Endpoint.objects.get_or_create(url=url4, protocol='unaffected', port=0, ip_version=0, is_dead=False) # make sure there are five endpoints: assert Endpoint.objects.all().count() == 6 # and duplicate scan results egs1, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 1', rating='1', rating_determined_on=timezone.now()) egs2, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 2', rating='2', rating_determined_on=timezone.now()) egs3, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 3', rating='3', rating_determined_on=timezone.now()) egs4, created = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 4', rating='4', rating_determined_on=timezone.now()) egs5, created = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 5', rating='5', rating_determined_on=timezone.now()) egs6, created = EndpointGenericScan.objects.get_or_create(endpoint=ep3, type='testscan 6', rating='6', rating_determined_on=timezone.now()) # some with the same data: egs7, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 1', rating='1', rating_determined_on=timezone.now()) egs8, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 2', rating='2', rating_determined_on=timezone.now()) egs9, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 3', rating='3', rating_determined_on=timezone.now()) egs10, created = EndpointGenericScan.objects.get_or_create(endpoint=ep6, type='unaffected', rating='3', rating_determined_on=timezone.now()) assert EndpointGenericScan.objects.all().count() == 10 # ep5 has no scans. # These urls, with their endpoints and scans are in three different lists. At the end of the test # all different lists should contain the same url, all other urls should be removed from the database. # the endpoints are merged into one set of unique endpoints. The scans are just attached to their respective # unique endpoint. urllist, created = UrlList.objects.all().get_or_create(name='test 1', account=account, scan_type='web') urllist.urls.add(url1) urllist.save() urllist, created = UrlList.objects.all().get_or_create(name='test 2', account=account, scan_type='web') urllist.urls.add(url2) urllist.save() urllist, created = UrlList.objects.all().get_or_create(name='test 3', account=account, scan_type='web') urllist.urls.add(url3) urllist.save() urllist, created = UrlList.objects.all().get_or_create(name='unaffected', account=account, scan_type='web') urllist.urls.add(url4) urllist.save() # now we're going to dedupe this whole thing. dedupe_urls() # assert all urls in the urllists are the same. list_1 = UrlList.objects.all().filter(name='test 1').first() list_2 = UrlList.objects.all().filter(name='test 2').first() list_3 = UrlList.objects.all().filter(name='test 3').first() assert list_1.urls.count() == list_2.urls.count() == list_3.urls.count() assert list_1.urls.first().id == list_2.urls.last().id == list_3.urls.first().id # verify that the unaffected list has a different url than the affected list list_4 = UrlList.objects.all().filter(name='unaffected').first() assert list_1.urls.first().id != list_4.urls.last().id # assert that there is only one url in each of these lists assert list_1.urls.count() == 1 assert list_2.urls.count() == 1 assert list_3.urls.count() == 1 assert list_4.urls.count() == 1 # assert there should only be one url left + the unaffected url, as the rest where identical assert Url.objects.all().filter(url='www.internet.nl').count() == 1 # the total number of urls should be two now, including one unaffected: assert Url.objects.all().count() == 1 + 1 # assert there should only be two different endpoints now, instead of 5, as they are deduped too + one unaffected assert Endpoint.objects.all().count() == 2 + 1 # assert all scan results are still saved assert EndpointGenericScan.objects.all().count() == 9 + 1