def _add_to_urls_to_urllist(account: Account, current_list: UrlList,
urls: List[str]) -> Dict[str, Any]:
counters: Dict[str, int] = {'added_to_list': 0, 'already_in_list': 0}
for url in urls:
# if already in list, don't need to save it again
already_in_list = UrlList.objects.all().filter(
account=account, id=current_list.id,
urls__url__iexact=url).exists()
if already_in_list:
counters['already_in_list'] += 1
continue
# if url already in database, we only need to add it to the list:
existing_url = Url.objects.all().filter(url=url).first()
if existing_url:
current_list.urls.add(existing_url)
counters['added_to_list'] += 1
else:
new_url = Url.add(url)
# always try to find a few dns endpoints...
compose_discover_task(urls_filter={'pk': new_url.id}).apply_async()
current_list.urls.add(new_url)
counters['added_to_list'] += 1
return counters
def add_url(new_url_string: str):
new_url = Url()
new_url.url = new_url_string
new_url.created_on = timezone.now()
new_url.save()
# always try to find a few dns endpoints...
compose_discover_task(urls_filter={'pk': new_url.id}).apply_async()
return new_url
def create_dashboard_scan_task(account: Account, urllist: UrlList,
save_as_scan_type: str,
endpoint_type: str) -> Task:
# The scan name is arbitrary. Add a lot of info to it so the scan can be tracked.
# A UUID will be added during registering
scan_name = "{'source': 'Internet.nl Dashboard', 'type': '%s', 'account': '%s', 'list': '%s'}" % (
save_as_scan_type, account.name, urllist.name)
api_url = API_URL_WEB if save_as_scan_type == 'web' else API_URL_MAIL
return (
# Should we only try to get the specifically needed dns_endpoint? At what volume we should / must?
# This discovers dns_endpoints. On the basis of this we know what urls we should scan an which
# ones we should not. We'll only scan if there are valid endpoint, just like at internet.nl
compose_discover_task(
**{
'urls_filter': {
'urls_in_dashboard_list': urllist,
'is_dead': False,
'not_resolvable': False
}
})
# Make sure that the discovery as listed above is actually used in the scan
| get_relevant_urls.si(urllist, endpoint_type)
# The urls are registered as part of the scan
| register_scan.s(account.internet_nl_api_username,
account.decrypt_password(), save_as_scan_type,
api_url, scan_name)
# When the scan is created, the scan is connected to the account for tracking purposes.
# This is visualized in the scan monitor.
| connect_scan_to_account.s(account, urllist))
def discovering_endpoints(scan: AccountInternetNLScan):
# Always immediately update the current state, so the amount of double calls is minimal:
# "discovered endpoints" to "discovering endpoints" and cause an infinte loop.
update_state("discovering endpoints", scan)
return (dns_endpoints.compose_discover_task(
**{
'urls_filter': {
'urls_in_dashboard_list': scan.urllist,
'is_dead': False,
'not_resolvable': False
}
})
| update_state.si("discovered endpoints", scan))
def discovering_endpoints(scan_id: int):
# Always immediately update the current state, so the amount of double calls is minimal:
# "discovered endpoints" to "discovering endpoints" and cause an infinte loop.
update_state("discovering endpoints", scan_id)
scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first()
if not scan:
log.warning(
f'Trying to discovering_endpoints with unknown scan: {scan_id}.')
return group([])
return (dns_endpoints.compose_discover_task(
**{
'urls_filter': {
'urls_in_dashboard_list_2__id': scan.urllist.id,
'is_dead': False,
'not_resolvable': False
}
})
| update_state.si("discovered endpoints", scan.id))