def _add_to_urls_to_urllist(account: Account, current_list: UrlList, urls: List[str]) -> Dict[str, Any]: counters: Dict[str, int] = {'added_to_list': 0, 'already_in_list': 0} for url in urls: # if already in list, don't need to save it again already_in_list = UrlList.objects.all().filter( account=account, id=current_list.id, urls__url__iexact=url).exists() if already_in_list: counters['already_in_list'] += 1 continue # if url already in database, we only need to add it to the list: existing_url = Url.objects.all().filter(url=url).first() if existing_url: current_list.urls.add(existing_url) counters['added_to_list'] += 1 else: new_url = Url.add(url) # always try to find a few dns endpoints... compose_discover_task(urls_filter={'pk': new_url.id}).apply_async() current_list.urls.add(new_url) counters['added_to_list'] += 1 return counters
def add_url(new_url_string: str): new_url = Url() new_url.url = new_url_string new_url.created_on = timezone.now() new_url.save() # always try to find a few dns endpoints... compose_discover_task(urls_filter={'pk': new_url.id}).apply_async() return new_url
def create_dashboard_scan_task(account: Account, urllist: UrlList, save_as_scan_type: str, endpoint_type: str) -> Task: # The scan name is arbitrary. Add a lot of info to it so the scan can be tracked. # A UUID will be added during registering scan_name = "{'source': 'Internet.nl Dashboard', 'type': '%s', 'account': '%s', 'list': '%s'}" % ( save_as_scan_type, account.name, urllist.name) api_url = API_URL_WEB if save_as_scan_type == 'web' else API_URL_MAIL return ( # Should we only try to get the specifically needed dns_endpoint? At what volume we should / must? # This discovers dns_endpoints. On the basis of this we know what urls we should scan an which # ones we should not. We'll only scan if there are valid endpoint, just like at internet.nl compose_discover_task( **{ 'urls_filter': { 'urls_in_dashboard_list': urllist, 'is_dead': False, 'not_resolvable': False } }) # Make sure that the discovery as listed above is actually used in the scan | get_relevant_urls.si(urllist, endpoint_type) # The urls are registered as part of the scan | register_scan.s(account.internet_nl_api_username, account.decrypt_password(), save_as_scan_type, api_url, scan_name) # When the scan is created, the scan is connected to the account for tracking purposes. # This is visualized in the scan monitor. | connect_scan_to_account.s(account, urllist))
def discovering_endpoints(scan: AccountInternetNLScan): # Always immediately update the current state, so the amount of double calls is minimal: # "discovered endpoints" to "discovering endpoints" and cause an infinte loop. update_state("discovering endpoints", scan) return (dns_endpoints.compose_discover_task( **{ 'urls_filter': { 'urls_in_dashboard_list': scan.urllist, 'is_dead': False, 'not_resolvable': False } }) | update_state.si("discovered endpoints", scan))
def discovering_endpoints(scan_id: int): # Always immediately update the current state, so the amount of double calls is minimal: # "discovered endpoints" to "discovering endpoints" and cause an infinte loop. update_state("discovering endpoints", scan_id) scan = AccountInternetNLScan.objects.all().filter(id=scan_id).first() if not scan: log.warning( f'Trying to discovering_endpoints with unknown scan: {scan_id}.') return group([]) return (dns_endpoints.compose_discover_task( **{ 'urls_filter': { 'urls_in_dashboard_list_2__id': scan.urllist.id, 'is_dead': False, 'not_resolvable': False } }) | update_state.si("discovered endpoints", scan.id))