def setup_test():
user = User(
**{
'first_name': 'test',
'last_name': 'test',
'username': '******',
'is_active': True
})
user.save()
account = Account(**{'name': 'test'})
account.save()
dashboarduser = DashboardUser(
**{
'mail_preferred_mail_address': '*****@*****.**',
'mail_preferred_language': 'nl',
'mail_send_mail_after_scan_finished': True,
'account': account,
'user': user
})
dashboarduser.save()
urllist = UrlList(**{'name': '', 'account': account})
urllist.save()
urllistreport = UrlListReport(
**{
'urllist': urllist,
'average_internet_nl_score': 42.42,
'at_when': timezone.now()
})
urllistreport.save()
internetnlv2scan = InternetNLV2Scan(**{
'type': 'web',
'scan_id': '123',
'state': 'finished'
})
internetnlv2scan.save()
accountinternetnlscan = AccountInternetNLScan(
**{
'account': account,
'scan': internetnlv2scan,
'urllist': urllist,
'started_on': timezone.now(),
'report': urllistreport,
'finished_on': timezone.now() + timedelta(hours=2)
})
accountinternetnlscan.save()
# first template:
template = EmailTemplate()
template.name = "scan_finished_en"
template.subject = "test"
template.description = "test"
template.email_html_text = "test {{report_average_internet_nl_score}}."
template.save()
def test_urllistreport_get_previous_report(db):
account = Account(**{'name': 'test'})
account.save()
u = UrlList(**{'name': '', 'account': account})
u.save()
urllistreport1 = UrlListReport(
**{
'urllist': u,
'average_internet_nl_score': 1,
'at_when': datetime(2020, 5, 1)
})
urllistreport1.save()
urllistreport2 = UrlListReport(
**{
'urllist': u,
'average_internet_nl_score': 1,
'at_when': datetime(2020, 5, 2)
})
urllistreport2.save()
urllistreport3 = UrlListReport(
**{
'urllist': u,
'average_internet_nl_score': 1,
'at_when': datetime(2020, 5, 3)
})
urllistreport3.save()
urllistreport4 = UrlListReport(
**{
'urllist': u,
'average_internet_nl_score': 1,
'at_when': datetime(2020, 5, 4)
})
urllistreport4.save()
urllistreport5 = UrlListReport(
**{
'urllist': u,
'average_internet_nl_score': 1,
'at_when': datetime(2020, 5, 5)
})
urllistreport5.save()
assert urllistreport5.get_previous_report_from_this_list(
) == urllistreport4
assert urllistreport4.get_previous_report_from_this_list(
) == urllistreport3
assert urllistreport3.get_previous_report_from_this_list(
) == urllistreport2
assert urllistreport2.get_previous_report_from_this_list(
) == urllistreport1
assert urllistreport1.get_previous_report_from_this_list() is None
def test_password_storage(db) -> None:
# normal usage
secret_password = '******'
account, created = Account.objects.all().get_or_create(name="test")
account.internet_nl_api_password = account.encrypt_password(secret_password)
account.save()
# this does not perform a retrieval, so of course it's the same.
assert account.decrypt_password() == secret_password
# no password set, should throw a valueError
with pytest.raises(ValueError, match=r'.*not set.*'):
account, created = Account.objects.all().get_or_create(name="value error")
account.decrypt_password()
# can create at once:
account2 = Account()
account2.name = 'test 2'
account2.internet_nl_api_username = '******'
account2.internet_nl_api_password = Account.encrypt_password(secret_password)
account2.save()
assert account2.decrypt_password() == secret_password
# If the field is stored as CharField instead of BinaryField, the type has become a string.
# It's a bit hard to cast that string back into bytes.
# verify that when retrieving all accounts, the password fields are still bytes.
accounts = Account.objects.all().filter(name__in=['test', 'test 2'])
for account in accounts:
# assert type(account.internet_nl_api_password) is bytes
assert account.decrypt_password() == secret_password
def create_dashboard_scan_task(account: Account, urllist: UrlList,
save_as_scan_type: str,
endpoint_type: str) -> Task:
# The scan name is arbitrary. Add a lot of info to it so the scan can be tracked.
# A UUID will be added during registering
scan_name = "{'source': 'Internet.nl Dashboard', 'type': '%s', 'account': '%s', 'list': '%s'}" % (
save_as_scan_type, account.name, urllist.name)
api_url = API_URL_WEB if save_as_scan_type == 'web' else API_URL_MAIL
return (
# Should we only try to get the specifically needed dns_endpoint? At what volume we should / must?
# This discovers dns_endpoints. On the basis of this we know what urls we should scan an which
# ones we should not. We'll only scan if there are valid endpoint, just like at internet.nl
compose_discover_task(
**{
'urls_filter': {
'urls_in_dashboard_list': urllist,
'is_dead': False,
'not_resolvable': False
}
})
# Make sure that the discovery as listed above is actually used in the scan
| get_relevant_urls.si(urllist, endpoint_type)
# The urls are registered as part of the scan
| register_scan.s(account.internet_nl_api_username,
account.decrypt_password(), save_as_scan_type,
api_url, scan_name)
# When the scan is created, the scan is connected to the account for tracking purposes.
# This is visualized in the scan monitor.
| connect_scan_to_account.s(account, urllist))
def test_per_acount_scanner(db) -> None:
account, created = Account.objects.all().get_or_create(
name="test",
internet_nl_api_username='******',
internet_nl_api_password=Account.encrypt_password('test'),
)
# some test urls
url1, created = Url.objects.all().get_or_create(url='www.internet.nl',
is_dead=False,
not_resolvable=False)
url2, created = Url.objects.all().get_or_create(url='internet.nl',
is_dead=False,
not_resolvable=False)
url3, created = Url.objects.all().get_or_create(url='nu.nl',
is_dead=False,
not_resolvable=False)
urls = [url1, url2, url3]
# a set of standard endpoints
ep = Endpoint(
**{
'url': urls[0],
'protocol': 'dns_a_aaaa',
'port': 0,
'ip_version': 0,
'is_dead': False
})
ep.save()
ep = Endpoint(
**{
'url': urls[1],
'protocol': 'dns_soa',
'port': 0,
'ip_version': 0,
'is_dead': False
})
ep.save()
# create an url list for this account:
urllist, created = UrlList.objects.all().get_or_create(name='test',
account=account,
scan_type='web')
for url in urls:
urllist.urls.add(url)
urllist.save()
# When filtering for an existing account, as created above, there should be two tasks created.
filters = {'account_filters': {'name': 'test'}}
scan_tasks = compose_task(**filters)
# two tasks: one for mail_dashboard, one for web, no type delivers one task but no content (should improve test).
assert len(scan_tasks.kwargs.get('tasks')) == 1
# This should not result in any scans, as the account does not exist, an empty group is returned.
filters = {'account_filters': {'name': 'not existing'}}
scan_tasks = compose_task(**filters)
log.debug(scan_tasks)
assert scan_tasks.kwargs.get('tasks') == []
def save_model(self, request, obj, form, change):
# If the internet_nl_api_password changed, encrypt the new value.
# Example usage and docs: https://github.com/pyca/cryptography
if 'new_password' in form.changed_data:
obj.internet_nl_api_password = Account.encrypt_password(form.cleaned_data.get('new_password'))
# check if the username / password combination is valid
super().save_model(request, obj, form, change)
def test_password_storage(db) -> None:
# normal usage
secret_password = '******'
account, created = Account.objects.all().get_or_create(name="test")
account.internet_nl_api_password = account.encrypt_password(
secret_password)
account.save()
# this does not perform a retrieval, so of course it's the same.
assert account.decrypt_password() == secret_password
# no password set, should throw a valueError
with pytest.raises(ValueError, match=r'.*not set.*'):
account, created = Account.objects.all().get_or_create(
name="value error")
account.decrypt_password()
with transaction.atomic():
# there is no magic going on. Passwords have to be encrypted manually.
# Warning: the transaction is broken here.
# An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
with pytest.raises(TypeError, match=r'.*bytes-like.*'):
account.internet_nl_api_password = "******"
account.save()
with pytest.raises(ValueError, match=r'.*not encrypted.*'):
account.decrypt_password()
# can create at once:
account2 = Account()
account2.name = 'test 2'
account2.internet_nl_api_username = '******'
account2.internet_nl_api_password = Account.encrypt_password(
secret_password)
account2.save()
assert account2.decrypt_password() == secret_password
# If the field is stored as CharField instead of BinaryField, the type has become a string.
# It's a bit hard to cast that string back into bytes.
# verify that when retrieving all accounts, the password fields are still bytes.
accounts = Account.objects.all().filter(name__in=['test', 'test 2'])
for account in accounts:
# assert type(account.internet_nl_api_password) is bytes
assert account.decrypt_password() == secret_password
def test_update_state(db):
account = Account()
account.save()
urllist = UrlList(**{'name': '', 'account': account})
urllist.save()
scan = AccountInternetNLScan()
scan.urllist = urllist
scan.account = account
scan.save()
update_state("new", scan.id)
# A situation has occurred where the log was already in the next step, but the scan state itself was
# at the old step. This caused the scan to block: as the logic was that if the log is in the correct / new state
# it was assumed the scan was also in that state. This is fixed and tested below.
scanlog = AccountInternetNLScanLog()
scanlog.scan = scan
scanlog.at_when = timezone.now()
scanlog.state = "out of sync"
scanlog.save()
update_state("out of sync", scan.id)
my_scan = AccountInternetNLScan.objects.all().first()
assert my_scan.state == "out of sync"
# new, out of sync, out of sync
# The last duplicate is stored to make it clear something went wrong. There is nothing 'attached' to the log
# other than understanding the process in case of weird situations.
assert AccountInternetNLScanLog.objects.all().count() == 3
# make sure the amount of log info does not grow if things are the same
update_state("out of sync", my_scan.id)
update_state("out of sync", my_scan.id)
update_state("out of sync", my_scan.id)
assert AccountInternetNLScanLog.objects.all().count() == 3
def save(self):
cleaned_data = super().clean()
username = cleaned_data.get("username")
password = cleaned_data.get("password")
user = User(**{'username': username})
user.set_password(password)
user.is_active = True
user.save()
account = Account(
**{
'name':
username,
'internet_nl_api_username':
username,
'internet_nl_api_password':
Account.encrypt_password(password),
'can_connect_to_internet_nl_api':
Account.connect_to_internet_nl_api(username, password)
})
account.save()
dashboarduser = DashboardUser(**{'user': user, 'account': account})
dashboarduser.save()
def save_instant_account(request) -> HttpResponse:
request = get_json_body(request)
username = request['username']
password = request['password']
if User.objects.all().filter(username=username).exists():
return JsonResponse(operation_response(error=True, message=f"User with username '{username}' already exists."))
if Account.objects.all().filter(name=username).exists():
return JsonResponse(operation_response(error=True,
message=f"Account with username {username}' already exists."))
# Extremely arbitrary password requirements. Just to make sure a password has been filled in.
if len(password) < 5:
return JsonResponse(operation_response(error=True, message=f"Password not filled in or not long enough."))
# all seems fine, let's add the user
user = User(**{'username': username})
user.set_password(password)
user.is_active = True
user.save()
account = Account(**{
'name': username,
'internet_nl_api_username': username,
'internet_nl_api_password': Account.encrypt_password(password),
'can_connect_to_internet_nl_api': Account.connect_to_internet_nl_api(username, password)
})
account.save()
dashboarduser = DashboardUser(**{'user': user, 'account': account})
dashboarduser.save()
return JsonResponse(operation_response(success=True, message=f"Account and user with name '{username}' created!"))
def test_retroactive_import(db, monkeypatch, redis_server):
monkeypatch.setattr(requests, 'get', mocked_requests_get)
account = Account()
account.id = 2
account.name = "test"
account.internet_nl_api_username = "******"
account.internet_nl_api_password = account.encrypt_password("test")
account.save()
retroactively_import({
'account_id': account.id,
'report_id': 'e738da28c0724e188dc808580fdcdf0e',
'scan_type': 'web'
})
assert Url.objects.all().count() == 18
assert Endpoint.objects.all().count() == 18
assert InternetNLScan.objects.all().count() == 1
assert AccountInternetNLScan.objects.all().count() == 1
def test_per_acount_scanner(db) -> None:
account, created = Account.objects.all().get_or_create(name="test",
internet_nl_api_username='******',
internet_nl_api_password=Account.encrypt_password('test'),)
# add duplicate (named) urls, the name is the same, the id is different.
url1, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False,
internal_notes='different data, is different id 1')
url2, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False,
internal_notes='different data, is different id 2')
url3, created = Url.objects.get_or_create(url='www.internet.nl', is_dead=False, not_resolvable=False,
internal_notes='different data, is different id 3')
url4, created = Url.objects.get_or_create(url='www.unaffected_url.nl', is_dead=False, not_resolvable=False,
internal_notes='different data, is different id 3')
# make sure that these three urls actually exist
assert Url.objects.all().count() == 4
# with duplicate endpoints, where the endpoints themselves are unique for the url.
ep1, created = Endpoint.objects.get_or_create(url=url1, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False)
ep2, created = Endpoint.objects.get_or_create(url=url2, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False)
ep3, created = Endpoint.objects.get_or_create(url=url3, protocol='dns_a_aaaa', port=0, ip_version=0, is_dead=False)
ep4, created = Endpoint.objects.get_or_create(url=url1, protocol='dns_soa', port=0, ip_version=0, is_dead=False)
ep5, created = Endpoint.objects.get_or_create(url=url2, protocol='dns_soa', port=0, ip_version=0, is_dead=False)
ep6, created = Endpoint.objects.get_or_create(url=url4, protocol='unaffected', port=0, ip_version=0, is_dead=False)
# make sure there are five endpoints:
assert Endpoint.objects.all().count() == 6
# and duplicate scan results
egs1, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 1', rating='1',
rating_determined_on=timezone.now())
egs2, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 2', rating='2',
rating_determined_on=timezone.now())
egs3, created = EndpointGenericScan.objects.get_or_create(endpoint=ep1, type='testscan 3', rating='3',
rating_determined_on=timezone.now())
egs4, created = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 4', rating='4',
rating_determined_on=timezone.now())
egs5, created = EndpointGenericScan.objects.get_or_create(endpoint=ep2, type='testscan 5', rating='5',
rating_determined_on=timezone.now())
egs6, created = EndpointGenericScan.objects.get_or_create(endpoint=ep3, type='testscan 6', rating='6',
rating_determined_on=timezone.now())
# some with the same data:
egs7, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 1', rating='1',
rating_determined_on=timezone.now())
egs8, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 2', rating='2',
rating_determined_on=timezone.now())
egs9, created = EndpointGenericScan.objects.get_or_create(endpoint=ep4, type='testscan 3', rating='3',
rating_determined_on=timezone.now())
egs10, created = EndpointGenericScan.objects.get_or_create(endpoint=ep6, type='unaffected', rating='3',
rating_determined_on=timezone.now())
assert EndpointGenericScan.objects.all().count() == 10
# ep5 has no scans.
# These urls, with their endpoints and scans are in three different lists. At the end of the test
# all different lists should contain the same url, all other urls should be removed from the database.
# the endpoints are merged into one set of unique endpoints. The scans are just attached to their respective
# unique endpoint.
urllist, created = UrlList.objects.all().get_or_create(name='test 1', account=account, scan_type='web')
urllist.urls.add(url1)
urllist.save()
urllist, created = UrlList.objects.all().get_or_create(name='test 2', account=account, scan_type='web')
urllist.urls.add(url2)
urllist.save()
urllist, created = UrlList.objects.all().get_or_create(name='test 3', account=account, scan_type='web')
urllist.urls.add(url3)
urllist.save()
urllist, created = UrlList.objects.all().get_or_create(name='unaffected', account=account, scan_type='web')
urllist.urls.add(url4)
urllist.save()
# now we're going to dedupe this whole thing.
dedupe_urls()
# assert all urls in the urllists are the same.
list_1 = UrlList.objects.all().filter(name='test 1').first()
list_2 = UrlList.objects.all().filter(name='test 2').first()
list_3 = UrlList.objects.all().filter(name='test 3').first()
assert list_1.urls.count() == list_2.urls.count() == list_3.urls.count()
assert list_1.urls.first().id == list_2.urls.last().id == list_3.urls.first().id
# verify that the unaffected list has a different url than the affected list
list_4 = UrlList.objects.all().filter(name='unaffected').first()
assert list_1.urls.first().id != list_4.urls.last().id
# assert that there is only one url in each of these lists
assert list_1.urls.count() == 1
assert list_2.urls.count() == 1
assert list_3.urls.count() == 1
assert list_4.urls.count() == 1
# assert there should only be one url left + the unaffected url, as the rest where identical
assert Url.objects.all().filter(url='www.internet.nl').count() == 1
# the total number of urls should be two now, including one unaffected:
assert Url.objects.all().count() == 1 + 1
# assert there should only be two different endpoints now, instead of 5, as they are deduped too + one unaffected
assert Endpoint.objects.all().count() == 2 + 1
# assert all scan results are still saved
assert EndpointGenericScan.objects.all().count() == 9 + 1