def new_scan(host, publish="off", start_new="on", all="done", ignoreMismatch="on"): """This function requests SSL Labs to run new scan for the target domain.""" if helpers.is_ip(host): print( red("[!] Your target host must be a domain, not an IP address! \ SSL Labs will onyl scan domains.")) exit() else: path = "analyze" payload = { 'host': host, 'publish': publish, 'start_new': start_new, 'all': all, 'ignoreMismatch': ignoreMismatch } results = request_api(path, payload) payload.pop('start_new') while results['status'] != 'READY' and results['status'] != 'ERROR': print("Scan in progress, please wait for the results.") time.sleep(30) results = request_api(path, payload) return results
def run_urlvoid_lookup(self, domain): """Collect reputation data from URLVoid for the target domain. This returns an ElementTree object. A free API key is required. """ if not helpers.is_ip(domain): try: if self.urlvoid_api_key != "": print( green("[+] Checking reputation for {} with URLVoid". format(domain))) url = "http://api.urlvoid.com/api1000/{}/host/{}"\ .format(self.urlvoid_api_key, domain) response = requests.get(url) tree = ET.fromstring(response.content) return tree else: print( green( "[-] No URLVoid API key, so skipping this test.")) return None except Exception as error: print(red("[!] Could not load URLVoid for reputation check!")) print(red("L.. Details: {}".format(error))) return None else: print( red("[!] Target is not a domain, so skipping URLVoid queries.") )
def prepare_scope(self, ip_list, domain_list, scope_file=None, domain=None): """Function to split the user's scope file into IP addresses and domain names.""" # Generate the scope lists from the supplied scope file, if there is one scope = [] if scope_file: scope = self.DC.generate_scope(scope_file) if domain: # Just in case the domain is not in the scope file, it's added here if not any(domain in d for d in scope): print( yellow( "[*] The provided domain, {}, was not found in your scope file, so \ it has been added to the scope for OSINT.".format(domain))) scope.append(domain) # Create lists of IP addresses and domain names from scope for item in scope: if helpers.is_ip(item): ip_list.append(item) elif item == "": pass else: domain_list.append(item) for target in scope: self.c.execute("INSERT INTO hosts VALUES (NULL,?,?,?)", (target, True, "Scope File")) self.conn.commit() return scope, ip_list, domain_list
def prepare_scope(self, scope_file, domain=None): """Function to split the user's scope file into IP addresses and domain names.""" # Generate the scope lists from the supplied scope file, if there is one scope = [] scope = self.DC.generate_scope(scope_file) if domain: # Just in case the domain is not in the scope file, it's added here if not any(domain in d for d in scope): print(yellow("[*] The provided domain, {}, was not found in your scope file, so \ it has been added to the scope for OSINT.".format(domain))) scope.append(domain) # Create lists of IP addresses and domain names from scope for item in scope: if helpers.is_ip(item): self.ip_addresses.append(item) elif item == "": pass else: self.domains_list.append(item) # Record the scope being used in the database self.c.execute('''CREATE TABLE 'ReportScope' ('Target' text)''') for target in scope: self.c.execute("INSERT INTO ReportScope VALUES (?)", (target,)) self.conn.commit() return scope, self.ip_addresses, self.domains_list
def run_urlvoid_lookup(self,domain): """Collect reputation data from URLVoid for the target domain. This returns an ElementTree object. A URLVoid API key is required. Parameters: domain The domain name to check with URLVoid """ if not helpers.is_ip(domain): try: if self.urlvoid_api_key != "": url = "http://api.urlvoid.com/api1000/{}/host/{}".format(self.urlvoid_api_key,domain) response = requests.get(url,timeout=self.requests_timeout) tree = ET.fromstring(response.content) return tree else: click.secho("[*] No URLVoid API key, so skipping this test.",fg="green") return None except Exception as error: click.secho("[!] Could not load URLVoid for reputation check!",fg="red") click.secho("L.. Details: {}".format(error),fg="red") return None else: click.secho("[!] Target is not a domain, so skipping URLVoid queries.",fg="red")
def run_urlvoid_lookup(self, domain): """Collect reputation data from URLVoid for the target domain. This returns an ElementTree object. A URLVoid API key is required. Parameters: domain The domain name to check with URLVoid """ if not helpers.is_ip(domain): try: if self.urlvoid_api_key != "": url = "http://api.urlvoid.com/api1000/{}/host/{}".format( self.urlvoid_api_key, domain) response = requests.get(url, timeout=self.requests_timeout) tree = ET.fromstring(response.content) return tree else: click.secho( "[*] No URLVoid API key, so skipping this test.", fg="green") return None except Exception as error: click.secho("[!] Could not load URLVoid for reputation check!", fg="red") click.secho("L.. Details: {}".format(error), fg="red") return None else: click.secho( "[!] Target is not a domain, so skipping URLVoid queries.", fg="red")
def lookup_robtex_ipinfo(self, ip_address): """Function to lookup information about a target IP address with Robtex.""" if helpers.is_ip(ip_address): request = requests.get(self.robtex_api + ip_address) ip_json = request.json() return ip_json else: print(red("[!] The provided IP for Robtex address is invalid!"))
def prepare_scope(scope_file, expanded_scope): """Parse IP ranges inside the provided scope file to expand IP ranges. This supports ranges with hyphens, underscores, and CIDRs. Parameters: scope_file A file containing domain name and IP addresses/ranges expanded_scope A list object for storing to expanded scope list """ try: with open(scope_file, "r") as scope_file: for target in scope_file: target = target.rstrip() # Record individual IPs and expand CIDRs if helpers.is_ip(target): ip_list = list(IPNetwork(target)) for address in sorted(ip_list): str_address = str(address) expanded_scope.append(str_address) # Sort IP ranges from domain names and expand the ranges if not helpers.is_domain(target): # Check for hyphenated ranges like those accepted by Nmap, e.g. 192.168.1.1-50 if "-" in target: target = target.rstrip() parts = target.split("-") startrange = parts[0] b = parts[0] dot_split = b.split(".") temp = "." # Join the values using a "." so it makes a valid IP combine = dot_split[0], dot_split[1], dot_split[ 2], parts[1] endrange = temp.join(combine) # Calculate the IP range ip_list = list(iter_iprange(startrange, endrange)) # Iterate through the range and remove ip_list for x in ip_list: temp = str(x) expanded_scope.append(temp) # Check if range has an underscore, e.g. 192.168.1.2_192.168.1.155 elif "_" in target: target = target.rstrip() parts = target.split("_") startrange = parts[0] endrange = parts[1] ip_list = list(iter_iprange(startrange, endrange)) for address in ip_list: str_address = str(address) expanded_scope.append(str_address) else: expanded_scope.append(target.rstrip()) click.secho( "[+] Scope list expanded to {} items. Proceeding with verification \ checks.".format(len(expanded_scope)), fg="green") except IOError as error: click.secho("[!] Parsing of scope file failed!", fg="red") click.secho("L.. Details: {}".format(error), fg="red")
def prepare_scope(scope_file,expanded_scope): """Parse IP ranges inside the provided scope file to expand IP ranges. This supports ranges with hyphens, underscores, and CIDRs. Parameters: scope_file A file containing domain name and IP addresses/ranges expanded_scope A list object for storing to expanded scope list """ try: with open(scope_file,"r") as scope_file: for target in scope_file: target = target.rstrip() # Record individual IPs and expand CIDRs if helpers.is_ip(target): ip_list = list(IPNetwork(target)) for address in sorted(ip_list): str_address = str(address) expanded_scope.append(str_address) # Sort IP ranges from domain names and expand the ranges if not helpers.is_domain(target): # Check for hyphenated ranges like those accepted by Nmap, e.g. 192.168.1.1-50 if "-" in target: target = target.rstrip() parts = target.split("-") startrange = parts[0] b = parts[0] dot_split = b.split(".") temp = "." # Join the values using a "." so it makes a valid IP combine = dot_split[0],dot_split[1],dot_split[2],parts[1] endrange = temp.join(combine) # Calculate the IP range ip_list = list(iter_iprange(startrange,endrange)) # Iterate through the range and remove ip_list for x in ip_list: temp = str(x) expanded_scope.append(temp) # Check if range has an underscore, e.g. 192.168.1.2_192.168.1.155 elif "_" in target: target = target.rstrip() parts = target.split("_") startrange = parts[0] endrange = parts[1] ip_list = list(iter_iprange(startrange,endrange)) for address in ip_list: str_address = str(address) expanded_scope.append(str_address) else: expanded_scope.append(target.rstrip()) click.secho("[+] Scope list expanded to {} items. Proceeding with verification \ checks.".format(len(expanded_scope)),fg="green") except IOError as error: click.secho("[!] Parsing of scope file failed!",fg="red") click.secho("L.. Details: {}".format(error),fg="red")
def perform_whois(expanded_scope, output): """Look-up the provided IP address in the ARIN database. Parameters: expanded_scope A list of domain name and IP addresses (no ranges) output A list object for storing the output """ total_addresses = len(expanded_scope) with click.progressbar(expanded_scope, label='Collecting information on addresses', length=total_addresses) as bar: for address in bar: if not helpers.is_ip(address): pass else: # Try to send GET request to the ARIN REST API for IP values try: r = requests.get("http://whois.arin.net/rest/ip/" + address + ".json", timeout=10) tmp = r.json() try: name = tmp['net']['customerRef']['@name'] # start = tmp['net']['netBlocks']['netBlock']['startAddress']['$'] # end = tmp['net']['netBlocks']['netBlock']['endAddress']['$'] hostname = reverse_lookup(address) cn = get_certificate(address) output[address] = address, name, hostname, cn except: # The formatting of ARIN data may change if an org is used for the contact name = tmp['net']['orgRef']['@name'] # start = tmp['net']['netBlocks']['netBlock']['startAddress']['$'] # end = tmp['net']['netBlocks']['netBlock']['endAddress']['$'] hostname = reverse_lookup(address) cn = get_certificate(address) output[address] = address, name, hostname, cn except: pass # Pause for just a sec to not destroy ARIN with requests sleep(1)
def results_from_cache(host, publish="off", start_new="off", from_cache="on", all="done"): """This function returns results from SSL Labs' cache (previously run scans).""" if helpers.is_ip(host): print( red("[!] Your target host must be a domain, not an IP address! \ SSL Labs will onyl scan domains.")) exit() else: path = "analyze" payload = { 'host': host, 'publish': publish, 'start_new': start_new, 'from_cache': from_cache, 'all': all } data = request_api(path, payload) return data
def _graph_hosts(self): """Convert the hosts table into Neo4j graph nodes.""" self.c.execute("SELECT host_address,in_scope_file,source FROM hosts") all_hosts = self.c.fetchall() for row in all_hosts: if row[1] == 0: scoped = False else: scoped = True if helpers.is_ip(row[0]): query = """ MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'}) RETURN x """ % (row[0], scoped, row[2]) helpers.execute_query(self.neo4j_driver, query) else: query = """ MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'}) RETURN x """ % (row[0], scoped, row[2]) helpers.execute_query(self.neo4j_driver, query)
def perform_whois(expanded_scope,output): """Look-up the provided IP address in the ARIN database. Parameters: expanded_scope A list of domain name and IP addresses (no ranges) output A list object for storing the output """ total_addresses = len(expanded_scope) with click.progressbar(expanded_scope, label='Collecting information on addresses', length=total_addresses) as bar: for address in bar: if not helpers.is_ip(address): pass else: # Try to send GET request to the ARIN REST API for IP values try: r = requests.get("http://whois.arin.net/rest/ip/" + address + ".json",timeout=10) tmp = r.json() try: name = tmp['net']['customerRef']['@name'] # start = tmp['net']['netBlocks']['netBlock']['startAddress']['$'] # end = tmp['net']['netBlocks']['netBlock']['endAddress']['$'] hostname = reverse_lookup(address) cn = get_certificate(address) output[address] = address,name,hostname,cn except: # The formatting of ARIN data may change if an org is used for the contact name = tmp['net']['orgRef']['@name'] # start = tmp['net']['netBlocks']['netBlock']['startAddress']['$'] # end = tmp['net']['netBlocks']['netBlock']['endAddress']['$'] hostname = reverse_lookup(address) cn = get_certificate(address) output[address] = address,name,hostname,cn except: pass # Pause for just a sec to not destroy ARIN with requests sleep(1)
def create_cymon_worksheet(self, target): """Function to check the provided the target against Cymon.io's database of threat feeds and then print the results. """ if helpers.is_ip(target): domains_results, ip_results = self.DC.search_cymon_ip(target) if domains_results: print(yellow("\n[+] Associated Domains:")) # Print out associated domains for the IP for result in domains_results: print("URL:\t %s" % result['name']) print("Created: %s" % result['created']) print("Updated: %s\n" % result['updated']) if ip_results: print(yellow("[+] Recorded Malicious Events:")) # Print out security events for the IP for result in ip_results: print("Title:\t\t %s" % result['title']) print("Description:\t %s" % result['description']) print("Created:\t %s" % result['created']) print("Updated:\t %s" % result['updated']) print("Details:\t %s\n" % result['details_url']) else: results = self.DC.search_cymon_domain(target) # Print out information for the domain if results: print(yellow("\n[+] Cymon.io events for %s" % target)) print("URL:\t %s" % results['name']) print("Created: %s" % results['created']) print("Updated: %s" % results['updated']) for source in results['sources']: print("Source:\t {}".format(source)) for ip in results['ips']: print("IP:\t {}".format(ip)) print("") print(green("[+] Cymon search completed!"))
def _graph_hosts(self): """Convert the hosts table into Neo4j graph nodes.""" self.c.execute("SELECT host_address,in_scope_file,source FROM hosts") all_hosts = self.c.fetchall() with click.progressbar(all_hosts, label="Creating Domain and IP nodes", length=len(all_hosts)) as bar: for row in bar: if row[1] == 0: scoped = False else: scoped = True if helpers.is_ip(row[0]): query = """ MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'}) RETURN x """ % (row[0], scoped, row[2]) helpers.execute_query(self.neo4j_driver, query) else: query = """ MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'}) RETURN x """ % (row[0], scoped, row[2]) helpers.execute_query(self.neo4j_driver, query)
def _graph_hosts(self): """Convert the hosts table into Neo4j graph nodes.""" self.c.execute("SELECT host_address,in_scope_file,source FROM hosts") all_hosts = self.c.fetchall() with click.progressbar(all_hosts, label="Creating Domain and IP nodes", length=len(all_hosts)) as bar: for row in bar: if row[1] == 0: scoped = False else: scoped = True if helpers.is_ip(row[0]): query = """ MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'}) RETURN x """% (row[0],scoped,row[2]) helpers.execute_query(self.neo4j_driver,query) else: query = """ MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'}) RETURN x """ % (row[0],scoped,row[2]) helpers.execute_query(self.neo4j_driver,query)
def create_domain_report_table(self, scope, ip_addresses, domains_list, verbose): """Function to generate a domain report consisting of information like DNS records and subdomains. """ # Create the DNS table for holding the domains' DNS records self.c.execute('''CREATE TABLE 'DNS' ('Domain' text, 'NSRecords' text, 'ARecords' text, 'MXRecords' text, 'TXTRecords' text, 'SOARecords' text, 'VulnerableCacheSnooping' text)''') # Get the DNS records for each domain for domain in domains_list: # Get the NS records try: temp = [] ns_records_list = self.DC.get_dns_record(domain, "NS") for rdata in ns_records_list.response.answer: for item in rdata.items: temp.append(item.to_text()) ns_records = ", ".join(temp) # Record name server that resolve cached queries vulnerable_dns_servers = [] for nameserver in temp: result = self.DC.check_dns_cache(nameserver.strip(".")) if result: vulnerable_dns_servers.append(result) except: ns_records = "None" # Get the A records try: temp = [] a_records = self.DC.get_dns_record(domain, "A") for rdata in a_records.response.answer: for item in rdata.items: temp.append(item.to_text()) a_records = ", ".join(temp) except: a_records = "None" # Get the MX records try: temp = [] mx_records = self.DC.get_dns_record(domain, "MX") for rdata in mx_records.response.answer: for item in rdata.items: temp.append(item.to_text()) mx_records = ", ".join(temp) except: mx_records = "None" # Get the TXT records try: temp = [] txt_records = self.DC.get_dns_record(domain, "TXT") for rdata in txt_records.response.answer: for item in rdata.items: temp.append(item.to_text()) txt_records = ", ".join(temp) except: txt_records = "None" # Get the SOA records try: temp = [] soa_records = self.DC.get_dns_record(domain, "SOA") for rdata in soa_records.response.answer: for item in rdata.items: temp.append(item.to_text()) soa_records = ", ".join(temp) except: soa_records = "None" # INSERT the DNS records into the table self.c.execute("INSERT INTO 'DNS' VALUES (?,?,?,?,?,?,?)", (domain, ns_records, a_records, mx_records, txt_records, soa_records, ", ".join(vulnerable_dns_servers))) self.conn.commit() sys.exit() # Create the Subdomains table for recording subdomain info for each domain self.c.execute('''CREATE TABLE 'Subdomains' ('Domain' text, 'Subdomain' text, 'IP' text, 'ASN' text, 'Provider' text, 'DomainFrontable' text)''') # Collect the subdomain information from DNS Dumpster and NetCraft for domain in domains_list: print(green("[+] Checking DNS Dumpster and NetCraft for {}".format(domain))) dumpster_results = [] netcraft_results = [] try: dumpster_results = self.DC.check_dns_dumpster(domain) except: print(red("[!] There was a problem collecting results from DNS Dumpster for {}.".format(domain))) try: netcraft_results = self.DC.check_netcraft(domain) except: print(red("[!] There was a problem collecting results from NetCraft for {}.".format(domain))) if dumpster_results: # See if we can save the domain map from DNS Dumpster if dumpster_results['image_data']: with open("reports/" + domain + "_Domain_Map.png", "wb") as fh: fh.write(base64.decodebytes(dumpster_results['image_data'])) # Record the info from DNS Dumpster for result in dumpster_results['dns_records']['host']: if result['reverse_dns']: # TODO: Reverse DNS subdomain = result['domain'] ip = result['ip'] asn = result['as'] provider = result['provider'] else: subdomain = result['domain'] ip = result['ip'] asn = result['as'] provider = result['provider'] # Check the subdomain for domain fronting possibilties frontable = self.DC.check_domain_fronting(result['domain']) # INSERT the subdomain info into the table self.c.execute("INSERT INTO Subdomains VALUES (?,?,?,?,?,?)", (domain, subdomain, ip, asn, provider, frontable)) self.conn.commit() # INSERT the subdomain info collected from NetCraft if netcraft_results: for result in netcraft_results: frontable = self.DC.check_domain_fronting(result) self.c.execute("INSERT INTO Subdomains VALUES (?,?,NULL,NULL,NULL,?)", (domain, result, frontable)) self.conn.commit() # Create IPHistory table for historical data collected from NetCraft self.c.execute('''CREATE TABLE 'IPHistory' ('Domain' text, 'Netblock Owner' text, 'IP' text)''') for domain in domains_list: ip_history = [] try: ip_history = self.DC.fetch_netcraft_domain_history(domain) except: print(red("[!] There was a problem collecting domain history from NetCraft for {}.".format(domain))) if ip_history: for result in ip_history: net_owner = result[0] ip = result[1] self.c.execute("INSERT INTO IPHistory VALUES (?,?,?)", (domain, net_owner, ip)) self.conn.commit() # Create the WhoisData table self.c.execute('''CREATE TABLE 'WhoisData' ('Domain' text, 'Registrar' text, 'Expiration' text, 'Organization' text, 'Registrant' text, 'AdminContact' text, 'TectContact' text, 'Address' text, 'DNSSec' text)''') # The whois lookups are only for domain names for domain in domains_list: try: # Run whois lookup print(green("[+] Running whois for {}".format(domain))) results = self.DC.run_whois(domain) # Log whois results to domain report if results: # Check if more than one expiration date is returned if isinstance(results['expiration_date'], datetime.date): expiration_date = results['expiration_date'] # We have a list, so break-up list into human readable dates and times else: expiration_date = [] for date in results['expiration_date']: expiration_date.append(date.strftime("%Y-%m-%d %H:%M:%S")) expiration_date = ", ".join(expiration_date) registrar = results['registrar'] org = results['org'] registrant = results['registrant'] admin_email = results['admin_email'] tech_email = results['tech_email'] address = results['address'].rstrip() dnssec = ', '.join(results['dnssec']) self.c.execute("INSERT INTO WhoisData VALUES (?,?,?,?,?,?,?,?,?)", (domain, registrar, expiration_date, org, registrant, admin_email, tech_email, address, dnssec)) self.conn.commit() except Exception as error: print(red("[!] There was an error running whois for {}!".format(domain))) print(red("L.. Details: {}".format(error))) # Create RDAP table self.c.execute('''CREATE TABLE 'RDAPData' ('IP' text, 'RDAPSource' text, 'Organization' text, 'NetworkCIDRs' text, 'ASN' text, 'CountryCode' text, 'RobtexRelatedDomains' text)''') # The RDAP lookups are only for IPs, but we get the IPs for each domain name, too for target in scope: try: # Slightly change output and recorded target if it's a domain if helpers.is_ip(target): target_ip = target for_output = target print(green("[+] Running RDAP lookup for {}".format(for_output))) elif target == "": pass else: target_ip = socket.gethostbyname(target) for_output = "{} ({})".format(target_ip, target) print(green("[+] Running RDAP lookup for {}".format(for_output))) # Log RDAP lookups results = self.DC.run_rdap(target_ip) if results: rdap_source = results['asn_registry'] org = results['network']['name'] net_cidr = results['network']['cidr'] asn = results['asn'] country_code = results['asn_country_code'] # TODO: Convert Verbose mode output into something easily recorded in the DB # # Verbose mode is optional to allow users to NOT be overwhelmed by contact data # if verbose: # row += 1 # for object_key, object_dict in results['objects'].items(): # if results['objects'] is not None: # for item in results['objects']: # name = results['objects'][item]['contact']['name'] # if name is not None: # dom_worksheet.write(row, 1, "Contact Name:") # dom_worksheet.write(row, 2, name) # row += 1 # title = results['objects'][item]['contact']['title'] # if title is not None: # dom_worksheet.write(row, 1, "Contact's Title:") # dom_worksheet.write(row, 2, title) # row += 1 # role = results['objects'][item]['contact']['role'] # if role is not None: # dom_worksheet.write(row, 1, "Contact's Role:") # dom_worksheet.write(row, 2, role) # row += 1 # email = results['objects'][item]['contact']['email'] # if email is not None: # dom_worksheet.write(row, 1, "Contact's Email:") # dom_worksheet.write(row, 2, email[0]['value']) # row += 1 # phone = results['objects'][item]['contact']['phone'] # if phone is not None: # dom_worksheet.write(row, 1, "Contact's Phone:") # dom_worksheet.write(row, 2, phone[0]['value']) # row += 1 # address = results['objects'][item]['contact']['address'] # if address is not None: # dom_worksheet.write(row, 1, "Contact's Address:") # dom_worksheet.write(row, 2, address[0]['value']) # row += 1 # Check Robtex for results for the current target robtex = self.DC.lookup_robtex_ipinfo(target_ip) if robtex: results = [] for result in robtex['pas']: results.append(result['o']) robtex_results = ", ".join(results) else: robtex_results = "None" self.c.execute("INSERT INTO RDAPData VALUES (?,?,?,?,?,?,?)", (for_output, rdap_source, org, net_cidr, asn, country_code, robtex_results)) self.conn.commit() except Exception as error: print(red("[!] The RDAP lookup failed for {}!".format(target))) print(red("L.. Details: {}".format(error))) # Create the URLVoid table self.c.execute('''CREATE TABLE 'URLVoidResults' ('Domain' text, 'IP' text, 'Hostname(s)' text, 'DomainAge' text, 'GoogleRank' text, 'AlexaRank' text, 'ASN' text, 'AsnName' text, 'MaliciousCount' text, 'MaliciousEngines' text)''') # Check each domain with URLVoid for reputation and some Alexa data for domain in domains_list: tree = self.DC.run_urlvoid_lookup(domain) count = "" engines = "" if tree is not None: # Check to see if urlvoid shows the domain flagged by any engines try: for child in tree: malicious_check = child.tag if malicious_check == "detections": detections = tree[1] engines = detections[0] count = ET.tostring(detections[1], method='text').rstrip().decode('ascii') temp = [] for engine in engines: temp.append(ET.tostring(engine, method='text').rstrip().decode('ascii')) engines = ", ".join(temp) print(yellow("[*] URLVoid found malicious activity reported for \ {}!".format(domain))) rep_data = tree[0] ip_data = rep_data[11] target = ET.tostring(rep_data[0], method='text').rstrip().decode('ascii') ip_add = ET.tostring(ip_data[0], method='text').rstrip().decode('ascii') hostnames = ET.tostring(ip_data[1], method='text').rstrip().decode('ascii') domain_age = ET.tostring(rep_data[3], method='text').rstrip().decode('ascii') google_rank = ET.tostring(rep_data[4], method='text').rstrip().decode('ascii') alexa_rank = ET.tostring(rep_data[5], method='text').rstrip().decode('ascii') asn = ET.tostring(ip_data[2], method='text').rstrip().decode('ascii') asn_name = ET.tostring(ip_data[3], method='text').rstrip().decode('ascii') self.c.execute("INSERT INTO URLVoidResults VALUES (?,?,?,?,?,?,?,?,?,?)", (target, ip_add, hostnames, domain_age, google_rank, alexa_rank, asn, asn_name, count, engines)) self.conn.commit() except: print(red("[!] There was an error getting the data for {}.".format(domain)))
def create_domain_report_table(self, scope, ip_list, domain_list, verbose): """Function to generate a domain report consisting of information like DNS records and subdomains. """ # Get the DNS records for each domain for domain in domain_list: vulnerable_dns_servers = [] # Get the NS records try: temp = [] ns_records_list = self.DC.get_dns_record(domain, "NS") for rdata in ns_records_list.response.answer: for item in rdata.items: temp.append(item.to_text()) ns_records = ", ".join(x.strip(".") for x in temp) # Record name server that resolve cached queries for nameserver in temp: result = self.DC.check_dns_cache(nameserver.strip(".")) if result: vulnerable_dns_servers.append(result) except: ns_records = "None" # Get the A records try: temp = [] a_records = self.DC.get_dns_record(domain, "A") for rdata in a_records.response.answer: for item in rdata.items: # Add A record IP to a temp list temp.append(item.to_text()) # Check if this a known IP and add it to hosts if not self.c.execute( "SELECT count(*) FROM hosts WHERE host_address=?", (item.to_text(), )) res = self.c.fetchone() if res[0] == 0: self.c.execute( "INSERT INTO 'hosts' VALUES (Null,?,?,?)", (item.to_text(), False, "Scope File Domain DNS")) self.conn.commit() # Also add it to our list of IP addresses ip_list.append(item.to_text()) a_records = ", ".join(temp) except: a_records = "None" # Get the MX records try: temp = [] mx_records = self.DC.get_dns_record(domain, "MX") for rdata in mx_records.response.answer: for item in rdata.items: temp.append(item.to_text()) mx_records = ", ".join(x.strip(".") for x in temp) except: mx_records = "None" # Get the TXT records try: temp = [] txt_records = self.DC.get_dns_record(domain, "TXT") for rdata in txt_records.response.answer: for item in rdata.items: temp.append(item.to_text()) txt_records = ", ".join(temp) except: txt_records = "None" # Get the SOA records try: temp = [] soa_records = self.DC.get_dns_record(domain, "SOA") for rdata in soa_records.response.answer: for item in rdata.items: temp.append(item.to_text()) soa_records = ", ".join(temp) except: soa_records = "None" # Get the _DMARC TXT record try: temp = [] dmarc_record = self.DC.get_dns_record("_dmarc." + domain, "TXT") for rdata in dmarc_record.response.answer: for item in rdata.items: temp.append(item.to_text()) dmarc_record = ", ".join(temp) except: dmarc_record = "None" # INSERT the DNS records into the table self.c.execute( "INSERT INTO 'dns' VALUES (NULL,?,?,?,?,?,?,?,?)", (domain, ns_records, a_records, mx_records, txt_records, soa_records, dmarc_record, ", ".join(vulnerable_dns_servers))) self.conn.commit() # Collect the subdomain information from DNS Dumpster and NetCraft for domain in domain_list: dumpster_results = [] netcraft_results = [] try: dumpster_results = self.DC.check_dns_dumpster(domain) except: print( red("[!] There was a problem collecting results from DNS Dumpster for {}." .format(domain))) try: netcraft_results = self.DC.check_netcraft(domain) except: print( red("[!] There was a problem collecting results from NetCraft for {}." .format(domain))) if dumpster_results: # See if we can save the domain map from DNS Dumpster if dumpster_results['image_data']: with open("reports/" + domain + "_Domain_Map.png", "wb") as fh: fh.write( base64.decodebytes(dumpster_results['image_data'])) # Record the info from DNS Dumpster for result in dumpster_results['dns_records']['host']: if result['reverse_dns']: subdomain = result['domain'] ip = result['ip'] # asn = result['as'] # provider = result['provider'] else: subdomain = result['domain'] ip = result['ip'] # asn = result['as'] # provider = result['provider'] # Check if this a known IP and add it to hosts if not self.c.execute( "SELECT count(*) FROM hosts WHERE host_address=?", (result['ip'], )) res = self.c.fetchone() if res[0] == 0: self.c.execute( "INSERT INTO 'hosts' VALUES (Null,?,?,?)", (result['ip'], False, "DNS Dumpster")) self.conn.commit() # Also add it to our list of IP addresses ip_list.append(result['ip']) # Check the subdomain for domain fronting possibilties frontable = self.DC.check_domain_fronting(result['domain']) # INSERT the subdomain info into the table # self.c.execute("INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?,?,?)", # (domain, subdomain, ip, asn, provider, frontable, "DNS Dumpster")) self.c.execute( "INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)", (domain, subdomain, ip, frontable, "DNS Dumpster")) self.conn.commit() # INSERT the subdomain info collected from NetCraft if netcraft_results: for result in netcraft_results: if not result in dumpster_results['dns_records']['host']: try: ip_address = socket.gethostbyname(result) # Check if this a known IP and add it to hosts if not self.c.execute( "SELECT count(*) FROM hosts WHERE host_address=?", (ip_address, )) res = self.c.fetchone() if res[0] == 0: self.c.execute( "INSERT INTO 'hosts' VALUES (Null,?,?,?)", (ip_address, False, "Netcraft DNS")) self.conn.commit() # Also add it to our list of IP addresses ip_list.append(ip_address) except: ip_address = "Lookup Failed" frontable = self.DC.check_domain_fronting(result) self.c.execute( "INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)", (domain, result, ip_address, frontable, "Netcraft")) self.conn.commit() # Try to collect certificate data for the domain try: cert_data = self.DC.run_censys_search_cert(domain) for cert in cert_data['results']: self.c.execute( "INSERT INTO 'certificates' VALUES (NULL,?,?,?)", (domain, cert["parsed.subject_dn"], cert["parsed.issuer_dn"])) self.conn.commit() cert_subdomains = self.DC.parse_cert_subdomains(cert_data) for sub in cert_subdomains: if not sub in dumpster_results['dns_records'][ 'host'] or sub in netcraft_results: # Check for wildcard subdomains from certificates and ignore them for this if not "*" in sub: try: ip_address = socket.gethostbyname(sub) # Check if this a known IP and add it to hosts if not self.c.execute( "SELECT count(*) FROM hosts WHERE host_address=?", (ip_address, )) res = self.c.fetchone() if res[0] == 0: self.c.execute( "INSERT INTO 'hosts' VALUES (Null,?,?,?)", (ip_address, False, "Certificate Lookup")) self.conn.commit() # Also add it to our list of IP addresses ip_list.append(ip_address) except: ip_address = "Lookup Failed" frontable = self.DC.check_domain_fronting(sub) self.c.execute( "INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)", (domain, sub, ip_address, frontable, "Certificate")) self.conn.commit() except: pass # Take a break for Censys's rate limits sleep(self.sleep) for domain in domain_list: ip_history = [] try: ip_history = self.DC.fetch_netcraft_domain_history(domain) except: print( red("[!] There was a problem collecting domain history from NetCraft for {}." .format(domain))) if ip_history: for result in ip_history: net_owner = result[0] ip_address = result[1] self.c.execute( "INSERT INTO ip_history VALUES (NULL,?,?,?)", (domain, net_owner, ip_address)) self.conn.commit() # Check if this a known IP and add it to hosts if not self.c.execute( "SELECT count(*) FROM hosts WHERE host_address=?", (ip_address, )) res = self.c.fetchone() if res[0] == 0: self.c.execute( "INSERT INTO 'hosts' VALUES (Null,?,?,?)", (ip_address, False, "Netcraft Domain IP History")) self.conn.commit() # Also add it to our list of IP addresses ip_list.append(ip_address) # The whois lookups are only for domain names for domain in domain_list: try: # Run whois lookup results = self.DC.run_whois(domain) if results: # Check if more than one expiration date is returned if isinstance(results['expiration_date'], datetime.date): expiration_date = results['expiration_date'] # We have a list, so break-up list into human readable dates and times else: expiration_date = [] for date in results['expiration_date']: expiration_date.append( date.strftime("%Y-%m-%d %H:%M:%S")) expiration_date = ", ".join(expiration_date) registrar = results['registrar'] org = results['org'] registrant = results['registrant'] admin_email = results['admin_email'] tech_email = results['tech_email'] address = results['address'].rstrip() if results['dnssec'] == "unsigned": dnssec = results['dnssec'] else: dnssec = ', '.join(results['dnssec']) self.c.execute( "INSERT INTO whois_data VALUES (NULL,?,?,?,?,?,?,?,?,?)", (domain, registrar, expiration_date, org, registrant, admin_email, tech_email, address, dnssec)) self.conn.commit() except Exception as error: print( red("[!] There was an error running whois for {}!".format( domain))) print(red("L.. Details: {}".format(error))) # The RDAP lookups are only for IPs, but we get the IPs for each domain name, too self.c.execute("SELECT host_address FROM hosts") collected_hosts = self.c.fetchall() # for target in scope: for target in collected_hosts: try: # Slightly change output and record target if it's a domain target = target[0] if helpers.is_ip(target): target_ip = target # for_output = target elif target == "": pass else: target_ip = socket.gethostbyname(target) # for_output = "{} ({})".format(target_ip, target) # Log RDAP lookups results = self.DC.run_rdap(target_ip) if results: rdap_source = results['asn_registry'] org = results['network']['name'] net_cidr = results['network']['cidr'] asn = results['asn'] country_code = results['asn_country_code'] # TODO: Convert Verbose mode output into something easily recorded in the DB # # Verbose mode is optional to allow users to NOT be overwhelmed by contact data # if verbose: # row += 1 # for object_key, object_dict in results['objects'].items(): # if results['objects'] is not None: # for item in results['objects']: # name = results['objects'][item]['contact']['name'] # if name is not None: # dom_worksheet.write(row, 1, "Contact Name:") # dom_worksheet.write(row, 2, name) # row += 1 # title = results['objects'][item]['contact']['title'] # if title is not None: # dom_worksheet.write(row, 1, "Contact's Title:") # dom_worksheet.write(row, 2, title) # row += 1 # role = results['objects'][item]['contact']['role'] # if role is not None: # dom_worksheet.write(row, 1, "Contact's Role:") # dom_worksheet.write(row, 2, role) # row += 1 # email = results['objects'][item]['contact']['email'] # if email is not None: # dom_worksheet.write(row, 1, "Contact's Email:") # dom_worksheet.write(row, 2, email[0]['value']) # row += 1 # phone = results['objects'][item]['contact']['phone'] # if phone is not None: # dom_worksheet.write(row, 1, "Contact's Phone:") # dom_worksheet.write(row, 2, phone[0]['value']) # row += 1 # address = results['objects'][item]['contact']['address'] # if address is not None: # dom_worksheet.write(row, 1, "Contact's Address:") # dom_worksheet.write(row, 2, address[0]['value']) # row += 1 # Check Robtex for results for the current target robtex = self.DC.lookup_robtex_ipinfo(target_ip) if robtex: results = [] for result in robtex['pas']: results.append(result['o']) robtex_results = ", ".join(results) else: robtex_results = "None" self.c.execute( "INSERT INTO rdap_data VALUES (NULL,?,?,?,?,?,?,?)", (target_ip, rdap_source, org, net_cidr, asn, country_code, robtex_results)) self.conn.commit() except Exception as error: print(red("[!] The RDAP lookup failed for {}!".format(target))) print(red("L.. Details: {}".format(error)))