def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:")
except getopt.GetoptError:
usage()
sys.exit()
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
ports_scanning = False
takeover_check = False
google_dorking = False
limit = 500
all_ip = []
full = []
dnsserver = ""
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
"baidu", "bing", "crtsh", "censys", "cymon", "bingapi",
"dogpile", "google", "googleCSE", "virustotal", "threatcrowd",
"googleplus", "google-profiles", 'google-certificates',
"linkedin", "pgp", "twitter", "trello", "vhost", "yahoo",
"netcraft", "hunter", "all"
])
if set(engines).issubset(supportedengines):
print("found supported engines")
print(("[-] Starting harvesting process for domain: " + word +
"\n"))
for engineitem in engines:
if engineitem == "google":
print("[-] Searching in Google:")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
if engineitem == "netcraft":
print("[-] Searching in Netcraft:")
search = netcraft.search_netcraft(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
if engineitem == "google-certificates":
print(
"[-] Searching in Google Certificate transparency report.."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
all_hosts = search.get_domains()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
if engineitem == "threatcrowd":
print("[-] Searching in Threatcrowd:")
search = threatcrowd.search_threatcrowd(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
if engineitem == "virustotal":
print("[-] Searching in Virustotal:")
search = virustotal.search_virustotal(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
if engineitem == "crtsh":
print("[-] Searching in CRT.sh:")
search = crtsh.search_crtsh(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
if engineitem == "googleCSE":
print("[-] Searching in Google Custom Search:")
search = googleCSE.search_googleCSE(word, limit, start)
search.process()
search.store_results()
all_emails = search.get_emails()
db = stash.stash_manager()
all_hosts = search.get_hostnames()
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing:")
search = bingsearch.search_bing(word, limit, start)
if engineitem == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch..")
search = dogpilesearch.search_dogpile(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == "pgp":
print("[-] Searching in PGP key server..")
search = pgpsearch.search_pgp(word)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
elif engineitem == "yahoo":
print("[-] Searching in Yahoo..")
search = yahoosearch.search_yahoo(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == "baidu":
print("[-] Searching in Baidu..")
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
elif engineitem == "googleplus":
print("[-] Searching in Google+ ..")
search = googleplussearch.search_googleplus(
word, limit)
search.process()
people = search.get_people()
print("Users from Google+:")
print("====================")
db = stash.stash_manager()
db.store_all(word, people, 'name', 'googleplus')
for user in people:
print(user)
sys.exit()
elif engineitem == "twitter":
print("[-] Searching in Twitter ..")
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print("Users from Twitter:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "linkedin":
print("[-] Searching in Linkedin..")
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
print("Users from Linkedin:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "google-profiles":
print("[-] Searching in Google profiles..")
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
print("Users from Google profiles:")
print("---------------------------")
for users in people:
print(users)
sys.exit()
elif engineitem == "hunter":
print("[-] Searching in Hunter:")
from discovery import huntersearch
#import locally or won't work
search = huntersearch.search_hunter(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
elif engineitem == "censys":
print("[-] Searching in Censys:")
from discovery import censys
#import locally or won't work
search = censys.search_censys(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == "cymon":
print("[-] Searching in Cymon:")
from discovery import cymon
#import locally or won't work
search = cymon.search_cymon(word)
search.process()
all_emails = []
all_hosts = []
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == "trello":
print("[-] Searching in Trello:")
from discovery import trello
#import locally or won't work
search = trello.search_trello(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_urls()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'trello')
db.store_all(word, all_emails, 'email', 'trello')
for x in all_hosts:
print(x)
sys.exit()
elif engineitem == "all":
print(("Full harvest on " + word))
all_emails = []
all_hosts = []
print("[-] Searching in Google..")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print("[-] Searching in PGP Key server..")
search = pgpsearch.search_pgp(word)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
print("[-] Searching in Netcraft server..")
search = netcraft.search_netcraft(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print("[-] Searching in ThreatCrowd server..")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print("[-] Searching in CRTSH server..")
search = crtsh.search_crtsh(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
print("[-] Searching in Virustotal server..")
search = virustotal.search_virustotal(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
print("[-] Searching in Bing..")
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
#Clean up email list, sort and uniq
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print("[-] Searching in Hunter:")
from discovery import huntersearch
#import locally
search = huntersearch.search_hunter(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
print(
"[-] Searching in Google Certificate transparency report.."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
domains = search.get_domains()
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
print("[-] Searching in Censys:")
from discovery import censys
search = censys.search_censys(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'censys')
db.store_all(word, all_hosts, 'host', 'censys')
else:
usage()
print(
"Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all"
)
sys.exit()
#Results############################################################
print("\n\033[1;32;40mHarvesting results")
if (len(all_ip) == 0):
print("No IP addresses found")
else:
print("\033[1;33;40m \n[+] IP addresses found in search engines:")
print("------------------------------------")
for i in all_ip:
print(i)
print("\n\n[+] Emails found:")
print("------------------")
#Sanity check to see if all_emails and all_hosts is defined
try:
all_emails
except NameError:
print('No emails found as all_emails is not defined.')
sys.exit()
try:
all_hosts
except NameError:
print('No hosts found as all_hosts is not defined.')
sys.exit()
if all_emails == []:
print("No emails found")
else:
print(("\n".join(all_emails)))
print("\033[1;33;40m \n[+] Hosts found in search engines:")
print("------------------------------------")
if all_hosts == [] or all_emails is None:
print("No hosts found")
else:
total = len(all_hosts)
print(("\nTotal hosts: " + str(total) + "\n"))
all_hosts = sorted(set(all_hosts))
print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ")
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != "empty":
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print("\n\033[94m[-] Starting DNS brute force: \033[1;33;40m")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print("\n\033[94m[-] Hosts found after DNS brute force:")
print("---------------------------------------")
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
#Port Scanning #################################################
if ports_scanning == True:
print("\n\n\033[1;32;40m[-] Scanning ports (active):\n")
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != "empty":
print(("- Scanning : " + host))
ports = [80, 443, 22, 8080, 21]
try:
scan = port_scanner.port_scan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(("\t\033[91m Detected open ports: " +
','.join(str(e)
for e in openports) + "\033[1;32;40m"))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
#DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print("\n[+] Starting active queries:")
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print(("\033[94m[-]Performing reverse lookup in : " + range +
"\033[1;33;40m"))
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print("Hosts found after reverse lookup (in target domain):")
print("---------------------------------")
for xh in dnsrev:
print(xh)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion:")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print("\n[+] Hosts found after DNS TLD expansion:")
print("------------------------------------------")
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print("\n[+] Virtual hosts:")
print("------------------")
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + "\t" + x))
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost = sorted(set(vhost))
else:
pass
#Shodan search####################################################
shodanres = []
shodanvisited = []
if shodan == True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full == []:
print('No host to search, exiting.')
sys.exit()
for x in full:
try:
ip = x.split(":")[1]
if not shodanvisited.count(ip):
print(("\tSearching for: " + ip))
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
#time.sleep(2)
for res in results['data']:
shodanres.append(
str("%s:%s - %s - %s - %s," %
(res['ip_str'], res['port'], res['os'],
res['isp'])))
except Exception as e:
pass
print("\n [+] Shodan results:")
print("------------------")
for x in shodanres:
print(x)
else:
pass
###################################################################
# Here i need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
#Reporting#######################################################
if filename != "":
try:
#NEW REPORT BEGINS
print("NEW REPORTING BEGINS:")
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscandomain = db.getlatestscandomain(word)
from lib import statichtmlgenerator
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.generatedashboardcode(scanboarddata)
HTMLcode += generator.generatescandetailsdomain(
word, latestscandomain)
from lib import reportgraph
import datetime
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscandomain)
#HTMLcode += graph.drawscattergraph(word, latestscandata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open("report.html", "w")
Html_file.write(HTMLcode)
Html_file.close()
print("NEW REPORTING FINISHED:")
#NEW REPORT FINISHED
print("[+] Saving files...")
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print("Error creating the file")
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(":")
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(":")
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split("SAPO")
# print " res[0] " + res[0] # ip/host
# print " res[1] " + res[1] # banner/info
# print " res[2] " + res[2] # port
file.write('<shodan>')
#page.h3(res[0])
file.write('<host>' + res[0] + '</host>')
#page.a("Port :" + res[2])
file.write('<port>' + res[2] + '</port>')
#page.pre(res[1])
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
#page.pre(x)
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print("Files saved!")
except Exception as er:
print(("Error saving XML file: " + str(er)))
sys.exit()
def start(argv):
if len(sys.argv) < 4:
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:')
except getopt.GetoptError:
Core.usage()
sys.exit(1)
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
all_emails = []
all_hosts = []
all_ip = []
bingapi = 'yes'
dnsbrute = False
dnslookup = False
dnsserver = ""
dnstld = False
filename = ""
full = []
google_dorking = False
host_ip = []
limit = 500
ports_scanning = False
shodan = False
start = 0
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = False
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = 'basic'
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
'baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon',
'dogpile', 'duckduckgo', 'google', 'googleCSE',
'google-certificates', 'google-profiles', 'hunter', 'linkedin',
'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello',
'twitter', 'vhost', 'virustotal', 'yahoo', 'all'
])
if set(engines).issubset(supportedengines):
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
if engineitem == 'bingapi':
bingapi = 'yes'
else:
bingapi = 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'googleCSE':
print(
'\033[94m[*] Searching Google Custom Search. \033[0m'
)
try:
search = googleCSE.SearchGoogleCSE(
word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'google-certificates':
print(
'\033[94m[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == 'google-profiles':
print('\033[94m[*] Searching Google profiles. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.SearchHunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'pgp':
print('\033[94m[*] Searching PGP key server. \033[0m')
try:
search = pgpsearch.SearchPgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
except Exception:
pass
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
# vhost
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
print(('Full harvest on ' + word))
all_emails = []
all_hosts = []
try:
print('[*] Searching Baidu.')
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
print('[*] Searching Bing.')
bingapi = 'no'
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print('[*] Searching Censys.')
from discovery import censys
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print('[*] Searching CRT.sh.')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
# dogpile
print('[*] Searching DuckDuckGo.')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print('[*] Searching Google.')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
'[*] Searching Google Certificate transparency report.'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
try:
print('[*] Searching Google profiles.')
search = googlesearch.search_google(
word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name',
'google-profile')
print('\nUsers from Google profiles:')
print('---------------------------')
for users in people:
print(users)
except Exception:
pass
print('[*] Searching Hunter.')
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.SearchHunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
# linkedin
print('[*] Searching Netcraft.')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print('[*] Searching PGP key server.')
try:
search = pgpsearch.SearchPgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
except Exception:
pass
print('[*] Searching Threatcrowd.')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print('[*] Searching Trello.')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
try:
print('[*] Searching Twitter.')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print('\nUsers from Twitter:')
print('-------------------')
for user in people:
print(user)
except Exception:
pass
# vhost
print('[*] Searching VirusTotal.')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
print('[*] Searching Yahoo.')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
ips = sorted(ipaddress.ip_address(line.strip()) for line in all_ip)
print('\n'.join(map(str, ips)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != 'empty':
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls == []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
print('\n[*] URLs found: ' + str(total))
print('--------------------')
for url in sorted(list(set(trello_urls))):
print(url)
# DNS brute force
dnsres = []
if dnsbrute is True:
print('\n[*] Starting DNS brute force.')
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS brute force:')
print('-------------------------------------')
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port scanning
if ports_scanning is True:
print('\n\n[*] Scanning ports (active).\n')
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != 'empty':
print(('[*] Scanning ' + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(('\t[*] Detected open ports: ' +
','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(':')[0]
range = ip.split('.')
range[3] = '0/24'
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print('[*] Performing reverse lookup in ' + range)
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print('Hosts found after reverse lookup (in target domain):')
print('----------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan
shodanres = []
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
if shodan is True:
print('\n\n[*] Shodan DB search (passive):\n')
try:
for ip in host_ip:
print(('\tSearching for: ' + ip))
shodan = shodansearch.search_shodan()
rowdata = shodan.search_ip(ip)
time.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print('\n [*] Shodan results:')
print('-------------------')
print(printedtable)
except Exception as e:
print(
f'[!] Error occurred in theHarvester - Shodan search module: {e}'
)
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('NEW REPORTING BEGINS.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open('report.html', 'w')
Html_file.write(HTMLcode)
Html_file.close()
print('NEW REPORTING FINISHED!')
print('[*] Saving files.')
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print('Error creating the file.')
try:
filename = filename.split('.')[0] + '.xml'
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(':')
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(':')
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print('Files saved!')
except Exception as er:
print(f'Error saving XML file: {er}')
print('\n\n')
sys.exit(0)
def start(argv):
if len(sys.argv) < 4:
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:")
except getopt.GetoptError:
Core.usage()
sys.exit(1)
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
start = 0
host_ip = []
all_hosts = []
all_emails = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
ports_scanning = False
takeover_check = False
google_dorking = False
limit = 500
all_ip = []
full = []
trello_info = ([], False)
dnsserver = ""
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
'baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon',
'dogpile', 'google', 'googleCSE', 'google-certificates',
'google-profiles', 'hunter', 'linkedin', 'netcraft', 'pgp',
'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost',
'virustotal', 'yahoo', 'all'
])
if set(engines).issubset(supportedengines):
print(("[-] Target domain: " + word + "\n"))
for engineitem in engines:
if engineitem == "baidu":
print("[-] Searching in Baidu.")
try:
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing.")
try:
search = bingsearch.search_bing(word, limit, start)
if engineitem == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "censys":
print("[-] Searching in Censys.")
from discovery import censys
# Import locally or won't work
search = censys.search_censys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == "crtsh":
print("[-] Searching in CRT.sh.")
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == "cymon":
print("[-] Searching in Cymon.")
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch.")
search = dogpilesearch.search_dogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == "google":
print("[-] Searching in Google.")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == "googleCSE":
print("[-] Searching in Google Custom Search.")
try:
search = googleCSE.search_googleCSE(
word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "google-certificates":
print(
"[-] Searching in Google Certificate transparency report."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == "google-profiles":
print("[-] Searching in Google profiles.")
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
print("\nUsers from Google profiles:")
print("---------------------------")
for users in people:
print(users)
sys.exit()
elif engineitem == "hunter":
print("[-] Searching in Hunter.")
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.search_hunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "linkedin":
print("[-] Searching in Linkedin.")
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
print("\nUsers from Linkedin:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "netcraft":
print("[-] Searching in Netcraft.")
search = netcraft.search_netcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == "pgp":
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
except Exception:
pass
elif engineitem == 'securityTrails':
print("[-] Searching in SecurityTrails.")
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "threatcrowd":
print("[-] Searching in Threatcrowd.")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
elif engineitem == "trello":
print("[-] Searching in Trello.")
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == "twitter":
print("[-] Searching in Twitter.")
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print("\nUsers from Twitter:")
print("-------------------")
for user in people:
print(user)
sys.exit()
# vhost
elif engineitem == "virustotal":
print("[-] Searching in VirusTotal.")
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == "yahoo":
print("[-] Searching in Yahoo.")
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == "all":
print(("Full harvest on " + word))
all_emails = []
all_hosts = []
# baidu
print("[-] Searching in Bing.")
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print("[-] Searching in Censys.")
from discovery import censys
search = censys.search_censys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print("[-] Searching in CRTSH server.")
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
# dogpile
print("[-] Searching in Google.")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
"[-] Searching in Google Certificate transparency report."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
# googleplus
# google-certificates
# google-profiles
print("[-] Searching in Hunter.")
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.search_hunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
# linkedin
print("[-] Searching in Netcraft server.")
search = netcraft.search_netcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
except Exception:
pass
print("[-] Searching in ThreatCrowd server.")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print("[-] Searching in Trello.")
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
# twitter
# vhost
print("[-] Searching in VirusTotal server.")
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
# yahoo
else:
print("[!] Invalid source.\n\n")
sys.exit(1)
# Results
print("\n\033[1;32;40mHarvesting results")
if len(all_ip) == 0:
print("No IP addresses found.")
else:
print("\033[1;33;40m \n[+] IP addresses found:")
print("-----------------------")
print("Total IP addresses: " + str(len(all_ip)) + "\n")
for ip in sorted(list(set(all_ip))):
print(ip)
print("\n[+] Emails found:")
print("-----------------")
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print('No emails found as all_emails is not defined.')
sys.exit(1)
try:
all_hosts
except NameError:
print('No hosts found as all_hosts is not defined.')
sys.exit(1)
if all_emails == []:
print("No emails found.")
else:
print("Total emails: " + str(len(all_emails)) + "\n")
print(("\n".join(sorted(list(set(all_emails))))))
print("\033[1;33;40m \n[+] Hosts found:")
print("----------------")
if all_hosts == []:
print("No hosts found.")
else:
total = len(all_hosts)
print(("Total hosts: " + str(total) + "\n"))
all_hosts = sorted(list(set(all_hosts)))
for host in all_hosts:
print(host)
print("\033[94m \n[-] Resolving hostnames to IPs.\033[1;33;40m")
print("------------------------------")
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != "empty":
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True: # Indicates user selected Trello.
print("\033[1;33;40m \n[+] URLs found from Trello:")
print("--------------------------")
trello_urls = trello_info[0]
if trello_urls == []:
print('\nNo Trello URLs found.')
else:
total = len(trello_urls)
print(("\nTotal URLs: " + str(total) + "\n"))
for url in sorted(list(set(trello_urls))):
print(url)
# DNS Brute force
dnsres = []
if dnsbrute is True:
print("\n\033[94m[-] Starting DNS brute force. \033[1;33;40m")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print("\n\033[94m[-] Hosts found after DNS brute force:")
print("-------------------------------------")
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port Scanning
if ports_scanning is True:
print("\n\n\033[1;32;40m[-] Scanning ports (active).\n")
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != "empty":
print(("- Scanning " + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(("\t\033[91m Detected open ports: " +
','.join(str(e)
for e in openports) + "\033[1;32;40m"))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print("\n[+] Starting active queries.")
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print(("\033[94m[-] Performing reverse lookup in " + range +
"\033[1;33;40m"))
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print("Hosts found after reverse lookup (in target domain):")
print("----------------------------------------------------")
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion.")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print("\n[+] Hosts found after DNS TLD expansion:")
print("----------------------------------------")
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == "basic":
print("\n[+] Virtual hosts:")
print("------------------")
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + "\t" + x))
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan search
shodanres = []
shodanvisited = []
if shodan is True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full == []:
print('No host to search, exiting.')
sys.exit(1)
for x in full:
try:
ip = x.split(":")[1]
if not shodanvisited.count(ip):
print(("\tSearching for: " + ip))
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results['data']:
shodanres.append(
str("%s:%s - %s - %s - %s," %
(res['ip_str'], res['port'], res['os'],
res['isp'])))
except Exception as e:
pass
print("\n [+] Shodan results:")
print("-------------------")
for x in shodanres:
print(x)
else:
pass
# Here we need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print("NEW REPORTING BEGINS.")
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open("report.html", "w")
Html_file.write(HTMLcode)
Html_file.close()
print("NEW REPORTING FINISHED!")
print("[+] Saving files.")
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print("Error creating the file.")
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(":")
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(":")
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split("SAPO")
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print("Files saved!")
except Exception as er:
print(("Error saving XML file: " + str(er)))
sys.exit()