def start(argv):
if len(sys.argv) < 5:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "hl:d:b:f:")
except getopt.GetoptError:
usage()
sys.exit()
word = ""
engine = ""
filename = ""
limit = 100
all_hosts = []
full = []
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-b':
engine = arg
if engine not in ("baidu",):
usage()
print "Invalid search engine, try with: baidu, waiting to add ..."
sys.exit()
else:
pass
elif opt == '-f':
filename = arg
if engine == "baidu":
print "[-] Searching in Baidu:"
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_hosts = search.get_hostnames()
# ************* Results ******************
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
if not all_hosts:
print "No hosts found"
else:
all_hosts = sorted(set(all_hosts))
print "[-] Resolving hostnames IPs... "
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
print host
# ************* Reporting ******************
if filename != "":
try:
print "\n[+] Saving files..."
html = htmlexport.HtmlExport(word, full, filename)
html.write_html()
except:
print traceback.print_exc()
print "Error creating the file"
# todo create xml report
sys.exit()
def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:vf:nhcte:")
except getopt.GetoptError:
usage()
sys.exit()
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
limit = 100
dnsserver = ""
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-t':
dnstld = True
elif opt == '-b':
engine = arg
if engine not in ("google", "googleCSE", "linkedin", "pgp", "all",
"google-profiles", "bing", "bingapi", "yandex",
"jigsaw", "dogpilesearch", "twitter",
"googleplus", "yahoo", "baidu"):
usage()
print "Invalid search engine, try with: bing, google, linkedin, pgp, jigsaw, bingapi, google-profiles, dogpilesearch, twitter, googleplus, yahoo, baidu"
sys.exit()
else:
pass
if engine == "google":
print "[-] Searching in Google:"
search = googlesearch.search_google(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
if engine == "googleCSE":
print "[-] Searching in Google Custom Search:"
search = googleCSE.search_googleCSE(word, limit, start)
search.process()
search.store_results()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
if engine == "exalead":
print "[-] Searching in Exalead:"
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "bing" or engine == "bingapi":
print "[-] Searching in Bing:"
search = bingsearch.search_bing(word, limit, start)
if engine == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "yandex": # Not working yet
print "[-] Searching in Yandex:"
search = yandexsearch.search_yandex(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "pgp":
print "[-] Searching in PGP key server.."
search = pgpsearch.search_pgp(word)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "jigsaw":
print "[-] Searching in Jigsaw.."
search = jigsaw.search_jigsaw(word, limit)
search.process()
people = search.get_people()
print "Users from Jigsaw:"
print "====================="
for user in people:
print user
sys.exit()
elif engine == "dogpilesearch":
print "[-] Searching in Dogpilesearch.."
search = dogpilesearch.search_dogpile(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "yahoo":
print "[-] Searching in Yahoo.."
search = yahoosearch.search_yahoo(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "baidu":
print "[-] Searching in Baidu.."
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "googleplus":
print "[-] Searching in Google+ .."
search = googleplussearch.search_googleplus(word, limit)
search.process()
people = search.get_people()
print "Users from Google+:"
print "===================="
for user in people:
print user
sys.exit()
elif engine == "twitter":
print "[-] Searching in Twitter .."
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
print "Users from Twitter:"
print "===================="
for user in people:
print user
sys.exit()
elif engine == "linkedin":
print "[-] Searching in Linkedin.."
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
print "Users from Linkedin:"
print "===================="
for user in people:
print user
sys.exit()
elif engine == "google-profiles":
print "[-] Searching in Google profiles.."
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
print "Users from Google profiles:"
print "---------------------------"
for users in people:
print users
sys.exit()
elif engine == "all":
print "Full harvest.."
all_emails = []
all_hosts = []
virtual = "basic"
print "[-] Searching in Google.."
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print "[-] Searching in PGP Key server.."
search = pgpsearch.search_pgp(word)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print "[-] Searching in Bing.."
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print "[-] Searching in Exalead.."
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
#Results############################################################
print "\n\n[+] Emails found:"
print "------------------"
if all_emails == []:
print "No emails found"
else:
for emails in all_emails:
print emails
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
if all_hosts == []:
print "No hosts found"
else:
print "[-] Resolving hostnames IPs... "
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[0]
print host
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print "\n[+] Starting active queries:"
analyzed_ranges = []
for x in full:
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
range = string.join(range, '.')
if not analyzed_ranges.count(range):
print "[-]Performing reverse lookup in :" + range
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print "Hosts found after reverse lookup:"
print "---------------------------------"
for xh in dnsrev:
print xh
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print "\n[-] Starting DNS brute force:"
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print "\n[+] Hosts found after DNS brute force:\n"
for y in res:
print y
dnsres.append(y)
if y not in full:
full.append(y)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print "[-] Starting DNS TLD expansion:"
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print "\n[+] Hosts found after DNS TLD expansion:"
print "=========================================="
for y in res:
print y
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print "[+] Virtual hosts:"
print "=================="
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print l + "\t" + x
vhost.append(l + ":" + x)
full.append(l + ":" + x)
else:
pass
shodanres = []
shodanvisited = []
if shodan == True:
print "[+] Shodan Database search:"
for x in full:
print x
try:
ip = x.split(":")[0]
if not shodanvisited.count(ip):
print "\tSearching for: " + x
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results:
shodanres.append(x + "SAPO" + str(res['banner']) +
"SAPO" + str(res['port']))
except:
pass
print "[+] Shodan results:"
print "==================="
for x in shodanres:
print x.split("SAPO")[0] + ":" + x.split("SAPO")[1]
else:
pass
###################################################################
# Here i need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
print emails
print hosts
else:
pass
if filename != "":
try:
print "[+] Saving files..."
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print e
print "Error creating the file"
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in all_hosts:
file.write('<host>' + x + '</host>')
for x in vhost:
file.write('<vhost>' + x + '</vhost>')
file.write('</theHarvester>')
file.close
print "Files saved!"
except Exception as er:
print "Error saving XML file: " + er
sys.exit()
def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:vf:nhcte:")
except getopt.GetoptError:
usage()
sys.exit()
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
limit = 100
dnsserver = ""
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-t':
dnstld = True
elif opt == '-b':
engine = arg
if engine not in ("baidu", "bing", "crtsh","bingapi","dogpile", "google", "googleCSE","virustotal", "googleplus", "google-profiles","linkedin", "pgp", "twitter", "vhost", "yahoo","netcraft","all"):
usage()
print "Invalid search engine, try with: baidu,bing,google,linkedin,twitter,yahoo,"
sys.exit()
else:
pass
if engine == "google":
print "[-] Searching in Google:"
search = googlesearch.search_google(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "bing" or engine == "bingapi":
print "[-] Searching in Bing:"
search = bingsearch.search_bing(word, limit, start)
if engine == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "yahoo":
print "[-] Searching in Yahoo.."
search = yahoosearch.search_yahoo(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "baidu":
print "[-] Searching in Baidu.."
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "twitter":
print "[-] Searching in Twitter .."
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
print "Users from Twitter:"
print "-------------------"
for user in people:
print user
sys.exit()
elif engine == "linkedin":
print "[-] Searching in Linkedin.."
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
print "Users from Linkedin:"
print "-------------------"
for user in people:
print user
sys.exit()
elif engine == "all":
print "Full harvest.."
all_emails = []
all_hosts = []
virtual = "basic"
print "[-] Searching in Google.."
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print "[-] Searching in Bing.."
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
#Clean up email list, sort and uniq
all_emails=sorted(set(all_emails))
#Results############################################################
print "\n\n[+] Emails found:"
print "------------------"
if all_emails == []:
print "No emails found"
else:
print "\n".join(all_emails)
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
if all_hosts == []:
print "No hosts found"
else:
all_hosts=sorted(set(all_hosts))
print "[-] Resolving hostnames IPs... "
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[0]
print host
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print "\n[+] Starting active queries:"
analyzed_ranges = []
for x in full:
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
range = string.join(range, '.')
if not analyzed_ranges.count(range):
print "[-]Performing reverse lookup in :" + range
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print "Hosts found after reverse lookup:"
print "---------------------------------"
for xh in dnsrev:
print xh
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print "\n[-] Starting DNS brute force:"
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print "\n[+] Hosts found after DNS brute force:\n"
print "---------------------------------------"
for y in res:
print y
dnsres.append(y)
if y not in full:
full.append(y)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print "[-] Starting DNS TLD expansion:"
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print "\n[+] Hosts found after DNS TLD expansion:"
print "------------------------------------------"
for y in res:
print y
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print "[+] Virtual hosts:"
print "-----------------"
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*','',x)
x = re.sub('<','',x)
x = re.sub('>','',x)
print l + "\t" + x
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost=sorted(set(vhost))
else:
pass
shodanres = []
shodanvisited = []
if shodan == True:
print "[+] Shodan Database search:"
for x in full:
print x
try:
ip = x.split(":")[0]
if not shodanvisited.count(ip):
print "\tSearching for: " + x
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results:
shodanres.append(
x + "SAPO" + str(res['banner']) + "SAPO" + str(res['port']))
except:
pass
print "[+] Shodan results:"
print "------------------"
for x in shodanres:
print x.split("SAPO")[0] + ":" + x.split("SAPO")[1]
else:
pass
###################################################################
# Here i need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
print emails
print hosts
else:
pass
#Reporting#######################################################
if filename != "":
try:
print "[+] Saving files..."
html = htmlExport.htmlExport(
all_emails,
full,
vhost,
dnsres,
dnsrev,
filename,
word,
shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print e
print "Error creating the file"
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><MailSpid3r>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(":")
if len(x) == 2:
file.write('<host>' + '<ip>' + x[0] + '</ip><hostname>' + x[1] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(":")
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[0] + '</ip><hostname>' + x[1] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split("SAPO")
# print " res[0] " + res[0] # ip/host
# print " res[1] " + res[1] # banner/info
# print " res[2] " + res[2] # port
file.write('<shodan>')
#page.h3(res[0])
file.write('<host>' + res[0] + '</host>')
#page.a("Port :" + res[2])
file.write('<port>' + res[2] + '</port>')
#page.pre(res[1])
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis=sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
#page.pre(x)
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</MailSpid3r>')
file.flush()
file.close()
print "Files saved!"
except Exception as er:
print "Error saving XML file: " + er
sys.exit()
print "------------------"
if all_emails == []:
print "No emails found"
else:
print "\n".join(all_emails)
print("\033[1;33;40m \n[+] Hosts found in search engines:")
print "------------------------------------"
if all_hosts == []:
print "No hosts found"
else:
total = len(all_hosts)
print "\nTotal hosts: " + str(total) + "\n"
all_hosts = sorted(set(all_hosts))
print "\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n "
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print host
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#Port Scanning #################################################
if ports_scanning == True:
print("\n\n\033[1;32;40m[-] Scanning ports (Active):\n")
for x in full:
host = x.split(' : ')[1]
domain = x.split(' : ')[0]
if host != "empty":
def start(argv):
if len(sys.argv) < 4:
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, 'l:d:b:s:u:vf:nhcgpte:')
except getopt.GetoptError:
Core.usage()
sys.exit(1)
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
all_emails = []
all_hosts = []
all_ip = []
bingapi = 'yes'
dnsbrute = False
dnslookup = False
dnsserver = ""
dnstld = False
filename = ""
full = []
google_dorking = False
host_ip = []
limit = 500
ports_scanning = False
shodan = False
start = 0
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = False
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = 'basic'
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
'baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon',
'dogpile', 'duckduckgo', 'google', 'googleCSE',
'google-certificates', 'google-profiles', 'hunter', 'linkedin',
'netcraft', 'pgp', 'securityTrails', 'threatcrowd', 'trello',
'twitter', 'vhost', 'virustotal', 'yahoo', 'all'
])
if set(engines).issubset(supportedengines):
print(f'\033[94m[*] Target domain: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
if engineitem == 'bingapi':
bingapi = 'yes'
else:
bingapi = 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'googleCSE':
print(
'\033[94m[*] Searching Google Custom Search. \033[0m'
)
try:
search = googleCSE.SearchGoogleCSE(
word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'google-certificates':
print(
'\033[94m[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == 'google-profiles':
print('\033[94m[*] Searching Google profiles. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.SearchHunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'pgp':
print('\033[94m[*] Searching PGP key server. \033[0m')
try:
search = pgpsearch.SearchPgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
except Exception:
pass
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
# vhost
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
print(('Full harvest on ' + word))
all_emails = []
all_hosts = []
try:
print('[*] Searching Baidu.')
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
print('[*] Searching Bing.')
bingapi = 'no'
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print('[*] Searching Censys.')
from discovery import censys
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print('[*] Searching CRT.sh.')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
# dogpile
print('[*] Searching DuckDuckGo.')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print('[*] Searching Google.')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
'[*] Searching Google Certificate transparency report.'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
try:
print('[*] Searching Google profiles.')
search = googlesearch.search_google(
word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name',
'google-profile')
print('\nUsers from Google profiles:')
print('---------------------------')
for users in people:
print(users)
except Exception:
pass
print('[*] Searching Hunter.')
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.SearchHunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
# linkedin
print('[*] Searching Netcraft.')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print('[*] Searching PGP key server.')
try:
search = pgpsearch.SearchPgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
except Exception:
pass
print('[*] Searching Threatcrowd.')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print('[*] Searching Trello.')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
try:
print('[*] Searching Twitter.')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print('\nUsers from Twitter:')
print('-------------------')
for user in people:
print(user)
except Exception:
pass
# vhost
print('[*] Searching VirusTotal.')
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
print('[*] Searching Yahoo.')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
ips = sorted(ipaddress.ip_address(line.strip()) for line in all_ip)
print('\n'.join(map(str, ips)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != 'empty':
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls == []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
print('\n[*] URLs found: ' + str(total))
print('--------------------')
for url in sorted(list(set(trello_urls))):
print(url)
# DNS brute force
dnsres = []
if dnsbrute is True:
print('\n[*] Starting DNS brute force.')
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS brute force:')
print('-------------------------------------')
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port scanning
if ports_scanning is True:
print('\n\n[*] Scanning ports (active).\n')
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != 'empty':
print(('[*] Scanning ' + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(('\t[*] Detected open ports: ' +
','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(':')[0]
range = ip.split('.')
range[3] = '0/24'
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print('[*] Performing reverse lookup in ' + range)
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print('Hosts found after reverse lookup (in target domain):')
print('----------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan
shodanres = []
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
if shodan is True:
print('\n\n[*] Shodan DB search (passive):\n')
try:
for ip in host_ip:
print(('\tSearching for: ' + ip))
shodan = shodansearch.search_shodan()
rowdata = shodan.search_ip(ip)
time.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print('\n [*] Shodan results:')
print('-------------------')
print(printedtable)
except Exception as e:
print(
f'[!] Error occurred in theHarvester - Shodan search module: {e}'
)
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('NEW REPORTING BEGINS.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open('report.html', 'w')
Html_file.write(HTMLcode)
Html_file.close()
print('NEW REPORTING FINISHED!')
print('[*] Saving files.')
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print('Error creating the file.')
try:
filename = filename.split('.')[0] + '.xml'
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(':')
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(':')
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print('Files saved!')
except Exception as er:
print(f'Error saving XML file: {er}')
print('\n\n')
sys.exit(0)
def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:vf:nhcte:")
except getopt.GetoptError:
usage()
sys.exit()
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
limit = 100
dnsserver = False
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-t':
dnstld = True
elif opt == '-b':
engine = arg
if engine not in ("google", "baidu", "linkedin", "pgp", "all",
"google-profiles", "bing", "bing_api", "yandex",
"people123", "jigsaw", "baidu-google"):
usage()
print "Invalid search engine, try with: baidu,bing, google, linkedin, pgp, exalead, jigsaw, bing_api, people123, google-profiles,baidu-google"
sys.exit()
else:
pass
if engine == "google":
print "[-] Searching in Google:"
search = googlesearch.search_google(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
if engine == "baidu":
print "[-] Searching in Baidu:"
search = baidusearch.search_baidu(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
#print word,limit,start,all_hosts
if engine == "exalead":
print "[-] Searching in Exalead:"
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "bing" or engine == "bingapi":
print "[-] Searching in Bing:"
search = bingsearch.search_bing(word, limit, start)
if engine == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "yandex": # Not working yet
print "[-] Searching in Yandex:"
search = yandexsearch.search_yandex(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "pgp":
print "[-] Searching in PGP key server.."
search = pgpsearch.search_pgp(word)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "people123":
print "[-] Searching in 123People.."
search = people123.search_123people(word, limit)
search.process()
people = search.get_people()
print "Users from 123People:"
print "====================="
for user in people:
print user
sys.exit()
elif engine == "jigsaw":
print "[-] Searching in Jigsaw.."
search = jigsaw.search_jigsaw(word, limit)
search.process()
people = search.get_people()
print "Users from Jigsaw:"
print "====================="
for user in people:
print user
sys.exit()
elif engine == "linkedin":
print "[-] Searching in Linkedin.."
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
print "Users from Linkedin:"
print "===================="
for user in people:
print user
sys.exit()
elif engine == "google-profiles":
print "[-] Searching in Google profiles.."
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
print "Users from Google profiles:"
print "---------------------------"
for users in people:
print users
sys.exit()
elif engine == "baidu-google":
print "Baidu and google harvest.."
all_emails = []
all_hosts = []
virtual = "basic"
print "[-] Searching in Google.."
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print "[-] Searching in Baidu.."
search = baidusearch.search_baidu(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
elif engine == "all":
print "Full harvest.."
all_emails = []
all_hosts = []
virtual = "basic"
print "[-] Searching in Google.."
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print "[-] Searching in PGP Key server.."
search = pgpsearch.search_pgp(word)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print "[-] Searching in Bing.."
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print "[-] Searching in Exalead.."
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
#Results############################################################
print "\n[+] Emails found:"
print "------------------"
if all_emails == []:
print "No emails found"
else:
xemail = list(
set(all_emails
)) #add by ha.cker list(set) is use for Remove duplicate email
#for emails in all_emails:
for emails in xemail:
print emails
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
if all_hosts == []:
print "No hosts found"
else:
xhost = list(
set(all_hosts
)) #add by ha.cker list(set) is use for Remove duplicate host
#full_host=hostchecker.Checker(all_hosts)
full_host = hostchecker.Checker(xhost)
full = full_host.check()
for host in full:
ip = host.split(':')[0]
print host
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print "\n[+] Starting active queries:"
analyzed_ranges = []
for x in full:
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
range = string.join(range, '.')
if not analyzed_ranges.count(range):
print "[-]Performing reverse lookup in :" + range
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print "Hosts found after reverse lookup:"
print "---------------------------------"
for xh in dnsrev:
print xh
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print "[-] Starting DNS brute force:"
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print "[+] Hosts found after DNS brute force:\n"
for y in res:
print y
dnsres.append(y)
if y not in full:
full.append(y)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print "[-] Starting DNS TLD expansion:"
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print "\n[+] Hosts found after DNS TLD expansion:"
print "=========================================="
for y in res:
print y
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print "[+] Virtual hosts:"
print "=================="
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
print l + "\t" + x
vhost.append(l + ":" + x)
full.append(l + ":" + x)
else:
pass
shodanres = []
shodanvisited = []
if shodan == True:
print "[+] Shodan Database search:"
for x in full:
print x
try:
ip = x.split(":")[0]
if not shodanvisited.count(ip):
print "\tSearching for: " + x
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results:
shodanres.append(x + "SAPO" + str(res['banner']) +
"SAPO" + str(res['port']))
except:
pass
print "[+] Shodan results:"
print "==================="
for x in shodanres:
print x.split("SAPO")[0] + ":" + x.split("SAPO")[1]
else:
pass
###################################################################
#Here i need to add explosion mode.
#Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
print emails
print hosts
else:
pass
if filename != "":
try:
print "Saving file"
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
sys.exit()
except Exception, e:
print e
print "Error creating the file"
def start():
parser = argparse.ArgumentParser(
description=
'theHarvester is used to gather open source intelligence (OSINT) on a\n'
'company or domain.')
parser.add_argument('-d',
'--domain',
help='company name or domain to search',
required=True)
parser.add_argument('-l',
'--limit',
help='limit the number of search results, default=500',
default=500,
type=int)
parser.add_argument('-S',
'--start',
help='start with result number X, default=0',
default=0,
type=int)
parser.add_argument('-g',
'--google-dork',
help='use Google Dorks for Google search',
default=False,
action='store_true')
parser.add_argument(
'-p',
'--port-scan',
help=
'scan the detected hosts and check for Takeovers (21,22,80,443,8080) default=False, params=True',
default=False)
parser.add_argument('-s',
'--shodan',
help='use Shodan to query discovered hosts',
default=False,
action='store_true')
parser.add_argument(
'-v',
'--virtual-host',
help=
'verify host name via DNS resolution and search for virtual hosts params=basic, default=False',
default=False)
parser.add_argument('-e',
'--dns-server',
help='DNS server to use for lookup')
parser.add_argument(
'-t',
'--dns-tld',
help='perform a DNS TLD expansion discovery, default False',
default=False)
parser.add_argument(
'-n',
'--dns-lookup',
help='enable DNS server lookup, default=False, params=True',
default=False)
parser.add_argument('-c',
'--dns-brute',
help='perform a DNS brute force on the domain',
default=False,
action='store_true')
parser.add_argument('-f',
'--filename',
help='save the results to an HTML and/or XML file',
default='',
type=str)
parser.add_argument('-b',
'--source',
help='''baidu, bing, bingapi, censys, crtsh, cymon,
dogpile, duckduckgo, google,
google-certificates, hunter, intelx,
linkedin, netcraft, securityTrails, threatcrowd,
trello, twitter, vhost, virustotal, yahoo, all''')
args = parser.parse_args()
try:
db = stash.stash_manager()
db.do_init()
except Exception:
pass
all_emails = []
all_hosts = []
all_ip = []
dnsbrute = args.dns_brute
dnslookup = args.dns_lookup
dnsserver = args.dns_server
dnstld = args.dns_tld
filename = args.filename
full = []
google_dorking = args.google_dork
host_ip = []
limit = args.limit
ports_scanning = args.port_scan
shodan = args.shodan
start = args.start
takeover_check = False
trello_info = ([], False)
vhost = []
virtual = args.virtual_host
word = args.domain
if args.source is not None:
engines = set(map(str.strip, args.source.split(',')))
if set(engines).issubset(Core.get_supportedengines()):
print(f'\033[94m[*] Target: {word} \n \033[0m')
for engineitem in engines:
if engineitem == 'baidu':
print('\033[94m[*] Searching Baidu. \033[0m')
try:
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == 'bing' or engineitem == 'bingapi':
print('\033[94m[*] Searching Bing. \033[0m')
try:
search = bingsearch.SearchBing(word, limit, start)
bingapi = ''
if engineitem == 'bingapi':
bingapi += 'yes'
else:
bingapi += 'no'
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'censys':
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
# Import locally or won't work
search = censys.SearchCensys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == 'crtsh':
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == 'cymon':
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == 'dogpile':
try:
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
except Exception as e:
print(
f'\033[93m[!] An error occurred with Dogpile: {e} \033[0m'
)
elif engineitem == 'duckduckgo':
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
elif engineitem == 'google':
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == 'google-certificates':
print(
'\033[94m[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == 'hunter':
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'intelx':
print('\033[94m[*] Searching Intelx. \033[0m')
from discovery import intelxsearch
# Import locally or won't work.
try:
search = intelxsearch.SearchIntelx(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'intelx')
db.store_all(word, all_emails, 'email', 'intelx')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
elif engineitem == 'linkedin':
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found Linkedin.\n\n')
else:
print(f'\n[*] Users found: {len(people)}')
print('---------------------')
for user in sorted(list(set(people))):
print(user)
sys.exit(0)
elif engineitem == 'netcraft':
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == 'securityTrails':
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
elif engineitem == 'threatcrowd':
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
elif engineitem == 'trello':
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == 'twitter':
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
elif engineitem == 'virustotal':
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.SearchVirustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == 'yahoo':
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == 'all':
print(('[*] Full harvest on ' + word))
all_emails = []
all_hosts = []
try:
print('\033[94m[*] Searching Baidu. \033[0m')
search = baidusearch.SearchBaidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
try:
print('\033[94m[*] Searching Bing. \033[0m')
bingapi = 'no'
search = bingsearch.SearchBing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
except Exception:
pass
print('\033[94m[*] Searching Censys. \033[0m')
from discovery import censys
search = censys.SearchCensys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print('\033[94m[*] Searching CRT.sh. \033[0m')
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
print('\033[94m[*] Searching Cymon. \033[0m')
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
print('\033[94m[*] Searching Dogpile. \033[0m')
search = dogpilesearch.SearchDogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
print('\033[94m[*] Searching DuckDuckGo. \033[0m')
from discovery import duckduckgosearch
search = duckduckgosearch.SearchDuckDuckGo(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'duckduckgo')
db.store_all(word, all_hosts, 'host', 'duckduckgo')
print('\033[94m[*] Searching Google. \033[0m')
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
'\033[94m[*] Searching Google Certificate transparency report. \033[0m'
)
search = googlecertificates.SearchGoogleCertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
print('\033[94m[*] Searching Hunter. \033[0m')
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.SearchHunter(word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
print('\033[94m[*] Searching Intelx. \033[0m')
from discovery import intelxsearch
# Import locally or won't work.
try:
search = intelxsearch.SearchIntelx(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'intelx')
db.store_all(word, all_emails, 'email', 'intelx')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
print(e)
print('\033[94m[*] Searching Linkedin. \033[0m')
search = linkedinsearch.SearchLinkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
if len(people) == 0:
print('\n[*] No users found.\n\n')
else:
print('\n[*] Users found: ' + str(len(people)))
print('---------------------')
for user in sorted(list(set(people))):
print(user)
print('\033[94m[*] Searching Netcraft. \033[0m')
search = netcraft.SearchNetcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print('\033[94m[*] Searching SecurityTrails. \033[0m')
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey):
print(e)
else:
pass
print('\033[94m[*] Searching Threatcrowd. \033[0m')
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
except Exception:
pass
print('\033[94m[*] Searching Trello. \033[0m')
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
try:
print('\033[94m[*] Searching Twitter. \033[0m')
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print('\nUsers from Twitter:')
print('-------------------')
for user in people:
print(user)
except Exception:
pass
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
print('\033[94m[*] Searching VirusTotal. \033[0m')
search = virustotal.SearchVirustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
try:
print('\033[94m[*] Searching Yahoo. \033[0m')
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
except Exception as e:
print(f'An exception occurred in yahoo: {e}')
else:
print('\033[93m[!] Invalid source.\n\n \033[0m')
sys.exit(1)
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print(
'\n\n\033[93m[!] No emails found because all_emails is not defined.\n\n \033[0m'
)
sys.exit(1)
try:
all_hosts
except NameError:
print(
'\n\n\033[93m[!] No hosts found because all_hosts is not defined.\n\n \033[0m'
)
sys.exit(1)
# Results
if len(all_ip) == 0:
print('\n[*] No IPs found.')
else:
print('\n[*] IPs found: ' + str(len(all_ip)))
print('-------------------')
ips = sorted(
ipaddress.ip_address(line.strip()) for line in set(all_ip))
print('\n'.join(map(str, ips)))
if len(all_emails) == 0:
print('\n[*] No emails found.')
else:
print('\n[*] Emails found: ' + str(len(all_emails)))
print('----------------------')
print(('\n'.join(sorted(list(set(all_emails))))))
if len(all_hosts) == 0:
print('\n[*] No hosts found.\n\n')
else:
print('\n[*] Hosts found: ' + str(len(all_hosts)))
print('---------------------')
all_hosts = sorted(list(set(all_hosts)))
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != 'empty':
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True:
trello_urls = trello_info[0]
if trello_urls == []:
print('\n[*] No URLs found.')
else:
total = len(trello_urls)
print('\n[*] URLs found: ' + str(total))
print('--------------------')
for url in sorted(list(set(trello_urls))):
print(url)
# DNS brute force
dnsres = []
if dnsbrute is True:
print('\n[*] Starting DNS brute force.')
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS brute force:')
print('-------------------------------------')
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port scanning
if ports_scanning is True:
print('\n\n[*] Scanning ports (active).\n')
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != 'empty':
print(('[*] Scanning ' + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(('\t[*] Detected open ports: ' +
','.join(str(e) for e in openports)))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print('\n[*] Starting active queries.')
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(':')[0]
range = ip.split('.')
range[3] = '0/24'
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print('[*] Performing reverse lookup in ' + range)
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print('[*] Hosts found after reverse lookup (in target domain):')
print('--------------------------------------------------------')
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld is True:
print('[*] Starting DNS TLD expansion.')
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print('\n[*] Hosts found after DNS TLD expansion:')
print('----------------------------------------')
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == 'basic':
print('\n[*] Virtual hosts:')
print('------------------')
for l in host_ip:
search = bingsearch.SearchBing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + '\t' + x))
vhost.append(l + ':' + x)
full.append(l + ':' + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan
shodanres = []
if shodan is True:
import texttable
tab = texttable.Texttable()
header = [
'IP address', 'Hostname', 'Org', 'Services:Ports', 'Technologies'
]
tab.header(header)
tab.set_cols_align(['c', 'c', 'c', 'c', 'c'])
tab.set_cols_valign(['m', 'm', 'm', 'm', 'm'])
tab.set_chars(['-', '|', '+', '#'])
tab.set_cols_width([15, 20, 15, 15, 18])
host_ip = list(set(host_ip))
print('\033[94m[*] Searching Shodan. \033[0m')
try:
for ip in host_ip:
print(('\tSearching for ' + ip))
shodan = shodansearch.SearchShodan()
rowdata = shodan.search_ip(ip)
time.sleep(2)
tab.add_row(rowdata)
printedtable = tab.draw()
print(printedtable)
except Exception as e:
print(f'\033[93m[!] An error occurred with Shodan: {e} \033[0m')
else:
pass
# Here we need to add explosion mode.
# We have to take out the TLDs to do this.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print('\n[*] Reporting started.')
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.GraphGenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open('report.html', 'w')
Html_file.write(HTMLcode)
Html_file.close()
print('[*] Reporting finished.')
print('[*] Saving files.')
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print(
'\n\033[93m[!] An error occurred while creating the output file.\n\n \033[0m'
)
sys.exit(1)
try:
filename = filename.split('.')[0] + '.xml'
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(':')
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(':')
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split('SAPO')
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ':' + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print('[*] Files saved.')
except Exception as er:
print(
f'\033[93m[!] An error occurred while saving the XML file: {er} \033[0m'
)
print('\n\n')
sys.exit(0)
def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:")
except getopt.GetoptError:
usage()
sys.exit()
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
ports_scanning = False
takeover_check = False
google_dorking = False
limit = 500
full = []
dnsserver = ""
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
"baidu", "bing", "crtsh", "censys", "cymon", "bingapi",
"dogpile", "google", "googleCSE", "virustotal", "threatcrowd",
"googleplus", "google-profiles", 'google-certificates',
"linkedin", "pgp", "twitter", "trello", "vhost", "yahoo",
"netcraft", "hunter", "all"
])
if set(engines).issubset(supportedengines):
print("found supported engines")
print(("[-] Starting harvesting process for domain: " + word +
"\n"))
for engineitem in engines:
if engineitem == "google":
print("[-] Searching in Google:")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
for x in all_hosts:
try:
db = stash.stash_manager()
db.store(word, x, 'host', 'google')
except Exception as e:
print(e)
if engineitem == "netcraft":
print("[-] Searching in Netcraft:")
search = netcraft.search_netcraft(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
if engineitem == "google-certificates":
print(
"[-] Searching in Google Certificate transparency report.."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
all_hosts = search.get_domains()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
if engineitem == "threatcrowd":
print("[-] Searching in Threatcrowd:")
search = threatcrowd.search_threatcrowd(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'threatcrowd')
if engineitem == "virustotal":
print("[-] Searching in Virustotal:")
search = virustotal.search_virustotal(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
if engineitem == "crtsh":
print("[-] Searching in CRT.sh:")
search = crtsh.search_crtsh(word)
search.process()
all_hosts = search.get_hostnames()
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
if engineitem == "googleCSE":
print("[-] Searching in Google Custom Search:")
search = googleCSE.search_googleCSE(word, limit, start)
search.process()
search.store_results()
all_emails = search.get_emails()
db = stash.stash_manager()
all_hosts = search.get_hostnames()
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing:")
search = bingsearch.search_bing(word, limit, start)
if engineitem == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch..")
search = dogpilesearch.search_dogpile(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engineitem == "pgp":
print("[-] Searching in PGP key server..")
search = pgpsearch.search_pgp(word)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db = stash.stash_manager()
db.store_all(word, all_emails, 'emails', 'pgp')
elif engineitem == "yahoo":
print("[-] Searching in Yahoo..")
search = yahoosearch.search_yahoo(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'emails', 'yahoo')
elif engineitem == "baidu":
print("[-] Searching in Baidu..")
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'emails', 'baidu')
elif engineitem == "googleplus":
print("[-] Searching in Google+ ..")
search = googleplussearch.search_googleplus(
word, limit)
search.process()
people = search.get_people()
print("Users from Google+:")
print("====================")
for user in people:
print(user)
sys.exit()
elif engineitem == "twitter":
print("[-] Searching in Twitter ..")
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
print("Users from Twitter:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "linkedin":
print("[-] Searching in Linkedin..")
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
print("Users from Linkedin:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "google-profiles":
print("[-] Searching in Google profiles..")
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
print("Users from Google profiles:")
print("---------------------------")
for users in people:
print(users)
sys.exit()
elif engineitem == "hunter":
print("[-] Searching in Hunter:")
from discovery import huntersearch
#import locally or won't work
search = huntersearch.search_hunter(word, limit, start)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engineitem == "censys":
print("[-] Searching in Censys:")
from discovery import censys
#import locally or won't work
search = censys.search_censys(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == "cymon":
print("[-] Searching in Cymon:")
from discovery import cymon
#import locally or won't work
search = cymon.search_cymon(word)
search.process()
all_emails = []
all_hosts = []
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == "trello":
print("[-] Searching in Trello:")
from discovery import trello
#import locally or won't work
search = trello.search_trello(word, limit)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_urls()
for x in all_hosts:
print(x)
sys.exit()
elif engineitem == "all":
print(("Full harvest on " + word))
all_emails = []
all_hosts = []
print("[-] Searching in Google..")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print("[-] Searching in PGP Key server..")
search = pgpsearch.search_pgp(word)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'PGP')
print("[-] Searching in Netcraft server..")
search = netcraft.search_netcraft(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print("[-] Searching in ThreatCrowd server..")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails = []
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print("[-] Searching in CRTSH server..")
search = crtsh.search_crtsh(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
print("[-] Searching in Virustotal server..")
search = virustotal.search_virustotal(word)
search.process()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
print("[-] Searching in Bing..")
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
#Clean up email list, sort and uniq
all_emails = sorted(set(all_emails))
print("[-] Searching in Hunter:")
from discovery import huntersearch
#import locally
search = huntersearch.search_hunter(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
print(
"[-] Searching in Google Certificate transparency report.."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
domains = search.get_domains()
all_hosts.extend(domains)
print("[-] Searching in Cymon:")
from discovery import cymon
search = cymon.search_cymon(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
print("[-] Searching in Censys:")
from discovery import censys
search = censys.search_censys(word)
search.process()
all_emails = []
all_ip = search.get_ipaddresses()
all_hosts = search.get_hostnames()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'censys')
db.store_all(word, all_hosts, 'host', 'censys')
else:
usage()
print(
"Invalid search engine, try with: baidu, bing, bingapi, crtsh, censys, cymon, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, hunter, all"
)
sys.exit()
#Results############################################################
print("\n\033[1;32;40mHarvesting results")
if (len(all_ip) == 0):
print("No IP addresses found")
else:
print("\033[1;33;40m \n[+] IP addresses found in search engines:")
print("------------------------------------")
for i in all_ip:
print(i)
print("\n\n[+] Emails found:")
print("------------------")
#Sanity check to see if all_emails and all_hosts is defined
try:
all_emails
except NameError:
print('No emails found as all_emails is not defined.')
sys.exit()
try:
all_hosts
except NameError:
print('No hosts found as all_hosts is not defined.')
sys.exit()
if all_emails == []:
print("No emails found")
else:
print(("\n".join(all_emails)))
print("\033[1;33;40m \n[+] Hosts found in search engines:")
print("------------------------------------")
if all_hosts == [] or all_emails is None:
print("No hosts found")
else:
total = len(all_hosts)
print(("\nTotal hosts: " + str(total) + "\n"))
all_hosts = sorted(set(all_hosts))
print("\033[94m[-] Resolving hostnames IPs...\033[1;33;40m \n ")
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != "empty":
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print("\n\033[94m[-] Starting DNS brute force: \033[1;33;40m")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print("\n\033[94m[-] Hosts found after DNS brute force:")
print("---------------------------------------")
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
#Port Scanning #################################################
if ports_scanning == True:
print("\n\n\033[1;32;40m[-] Scanning ports (active):\n")
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != "empty":
print(("- Scanning : " + host))
ports = [80, 443, 22, 8080, 21]
try:
scan = port_scanner.port_scan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(("\t\033[91m Detected open ports: " +
','.join(str(e)
for e in openports) + "\033[1;32;40m"))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
#DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print("\n[+] Starting active queries:")
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print(("\033[94m[-]Performing reverse lookup in : " + range +
"\033[1;33;40m"))
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print("Hosts found after reverse lookup (in target domain):")
print("---------------------------------")
for xh in dnsrev:
print(xh)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion:")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print("\n[+] Hosts found after DNS TLD expansion:")
print("------------------------------------------")
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print("\n[+] Virtual hosts:")
print("------------------")
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + "\t" + x))
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost = sorted(set(vhost))
else:
pass
#Shodan search####################################################
shodanres = []
shodanvisited = []
if shodan == True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full == []:
print('No host to search, exiting.')
sys.exit()
for x in full:
try:
ip = x.split(":")[1]
if not shodanvisited.count(ip):
print(("\tSearching for: " + ip))
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
#time.sleep(2)
for res in results['data']:
shodanres.append(
str("%s:%s - %s - %s - %s," %
(res['ip_str'], res['port'], res['os'],
res['isp'])))
except Exception as e:
pass
print("\n [+] Shodan results:")
print("------------------")
for x in shodanres:
print(x)
else:
pass
###################################################################
# Here i need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
#Reporting#######################################################
if filename != "":
try:
print("[+] Saving files...")
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print("Error creating the file")
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(":")
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(":")
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split("SAPO")
# print " res[0] " + res[0] # ip/host
# print " res[1] " + res[1] # banner/info
# print " res[2] " + res[2] # port
file.write('<shodan>')
#page.h3(res[0])
file.write('<host>' + res[0] + '</host>')
#page.a("Port :" + res[2])
file.write('<port>' + res[2] + '</port>')
#page.pre(res[1])
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
#page.pre(x)
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print("Files saved!")
except Exception as er:
print(("Error saving XML file: " + str(er)))
sys.exit()
def start(argv):
if len(sys.argv) < 4:
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:vf:nhcte:")
except getopt.GetoptError:
sys.exit()
start = 0
host_ip = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
limit = 100
dnsserver = ""
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-b':
engine = arg
if engine not in ("baidu", "bing", "crtsh", "bingapi", "dogpile",
"google", "googleCSE", "virustotal",
"googleplus", "google-profiles", "linkedin",
"pgp", "twitter", "vhost", "yahoo", "netcraft",
"all"):
usage()
print "Invalid search engine, try with: baidu, bing, bingapi,crtsh, dogpile, google, googleCSE, virustotal, netcraft, googleplus, google-profiles, linkedin, pgp, twitter, vhost, yahoo, all"
sys.exit()
else:
pass
if engine == "all":
all_emails = []
all_hosts = []
virtual = "basic"
print '\033[1;97m[>]\033[1;m Initiating 3 intel modules'
print "\033[1;97m[>]\033[1;m Loading Alpha module (1/3)"
search = googlesearch.search_google(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print "\033[1;97m[>]\033[1;m Beta module deployed (2/3)"
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print "\033[1;97m[>]\033[1;m Gamma module initiated (3/3)"
search = exaleadsearch.search_exalead(word, limit, start)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
#Clean up email list, sort and uniq
all_emails = sorted(set(all_emails))
#Results############################################################
if all_emails == []:
print "No emails found"
else:
print "\n\n[+] Emails found:"
print "------------------"
print "\n".join(all_emails)
if all_hosts == []:
print "No hosts found"
else:
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
all_hosts = sorted(set(all_hosts))
print "[-] Resolving hostnames IPs... "
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[0]
print host
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
#Virtual hosts search###############################################
if virtual == "basic":
print "[+] Virtual hosts:"
print "-----------------"
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print l + "\t" + x
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost = sorted(set(vhost))
else:
pass
def start(argv):
engine_string = ''
for engine_name, engine_function in engine_list.iteritems() :
engine_string += (engine_name+ ',')
parser = ArgumentParser(epilog=
"""\nExamples:
./theharvester.py -d microsoft.com -l 500 -b google
./theharvester.py -d microsoft.com -b pgp
./theharvester.py -d microsoft -l 200 -b linkedin -qvnct\n""", formatter_class=RawTextHelpFormatter)
parser.add_argument("-d", "--domain", dest="word", help="Domain or company name to search for")
parser.add_argument("-b", "--engine", dest="engine", help="Data source ("+engine_string+"all) (default google)",default="google")
parser.add_argument("-s", "--start", dest="start", type=int, help="Start in result number X (default 0)", default=0)
parser.add_argument("-v", "--virtual", dest="virtual", action="store_true", help="Verify host name via dns resolution and search for\nvirtual hosts")
parser.add_argument("-n", "--dns-lookup", dest="dns_lookup", action="store_true", help="Perform a DNS reverse query on all ranges discovered")
parser.add_argument("-c", "--dns-brute", dest="dns_brute", action="store_true", help="Perform a DNS brute force for the domain name (slow)")
parser.add_argument("-t", "--dns-tld", dest="dns_tld", action="store_true", help="Perform a DNS TLD expansion discovery")
parser.add_argument("-e", "--dns-server", dest="dns_server", help="Use this DNS server")
parser.add_argument("-l", "--limit", dest="limit", type=int, default=100, help="Limit the number of results to work with\n(bing goes from 50 to 50 result")
parser.add_argument("-q", "--shodan-lookup", dest="shodan_lookup", action="store_true", help="Use SHODAN database to query discovered hosts")
parser.add_argument("-o", "--output", dest="output", action="append", nargs=2, metavar=('[X|H]', '<filename>'),
help="-o H <html_filename>\tOutput to HTML file\n-o X <xml_filename>\tOutput to XML file")
options = parser.parse_args()
print_banner()
if not options.word:
parser.error ("Domain search is mandatory")
sys.exit()
if validate_output(options.output) == False:
parser.error("Invalid output options")
if options.engine != 'all' and options.engine not in engine_list:
parser.error ("Invalid search engine, try with: " + engine_string + 'all')
parser.print_help()
sys.exit()
search_results = perform_search(options.word, options)
search_results.remove_duplicates()
output_results = OutputItem
output_results.word = options.word
output_results.emails = search_results.emails
output_results.people = search_results.people
#Results############################################################
print "\n[+] Emails found:"
print "------------------"
if not search_results.emails:
print "No emails found"
else:
for emails in search_results.emails:
print emails
print "\n[+] People found:"
print "------------------"
if not search_results.people:
print "No people found"
else:
for person in search_results.people:
print person
output_results.search_hosts_ips = {}
print "\n[+] Hosts found in search engines:"
print "------------------------------------"
if not search_results.hostnames:
print "No hosts found"
else:
host_to_ip=hostchecker.Checker()
output_results.search_hosts_ips=host_to_ip.hosts_to_ips(search_results.hostnames)
if len(output_results.search_hosts_ips) == 0:
print "No hosts found"
else:
for host, ip in output_results.search_hosts_ips.iteritems() :
print ip+"\t"+host
unique_ips = []
if len(output_results.search_hosts_ips)>0:
unique_ips = list(set(output_results.search_hosts_ips.values()))
# We leave all_hosts_ips and unique_ips alone as they represent
# the results of the initial search. If we want to go deep we can
# always revisit them later and start adding more.
if True in (options.dns_lookup, options.dns_brute, options.dns_tld):
print "\n[+] Starting active queries:"
#DNS reverse lookup on a Class C########################################
output_results.dns_reverse_results = {}
if options.dns_lookup==True:
analyzed_ranges=[]
for ip in unique_ips:
class_c_range = ip_to_class_c_cidr(ip)
if range not in analyzed_ranges:
print "\n[-]Performing reverse lookup on: " + class_c_range
a=dnssearch.dns_reverse(class_c_range,True)
a.list()
output_results.dns_reverse_results=a.process()
analyzed_ranges.append(class_c_range)
else:
continue
print "\nHosts found after reverse lookup:"
print "---------------------------------"
if len(output_results.dns_reverse_results) == 0:
print ("None")
for host, ip in output_results.dns_reverse_results.iteritems() :
print host
#DNS Brute force####################################################
output_results.dns_brute_results = {}
if options.dns_brute==True:
print "[-] Starting DNS brute force:"
a=dnssearch.dns_force(options.word, options.dns_server, verbose=True)
output_results.dns_brute_results=a.process()
print "[+] Hosts found after DNS brute force:\n"
if len(output_results.dns_brute_results) == 0:
print ("None")
#all_hosts_ips[host] = ip
#DNS TLD expansion###################################################
output_results.dns_tld_results = {}
if options.dns_tld==True:
print "\n[-] Starting DNS TLD expansion:\n"
a=dnssearch.dns_tld(options.word,options.dns_server,verbose=True)
output_results.dns_tld_results=a.process()
print "[+] Hosts found after DNS TLD expansion:"
print "=========================================="
if len(output_results.dns_tld_results) == 0:
print ("None")
for host, ip in output_results.dns_tld_results.iteritems() :
print host
#Virtual hosts search###############################################
output_results.virtual_host_results = {}
if options.virtual == True:
print "\n[+] Virtual hosts:"
print "=================="
for ip in unique_ips:
search=bingsearch.search_bing(ip, options)
search.process_vhost()
hostnames_from_ip=search.get_allhostnames()
for host in hostnames_from_ip:
print ip+"\t"+host
# Store the virtual hosts
# Do we want to add these to the main hostlist? Maybe not
output_results.virtual_host_results[host] = ip
shodanvisited=[]
output_results.shodan_results = []
if options.shodan_lookup == True:
print "[+] Shodan Database search:"
for host, ip in output_results.search_hosts_ips.iteritems() :
try:
if not shodanvisited.count(ip):
print "\tSearching for: " + ip+": "+host
a=shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results=a.run()
for res in results:
output_results.shodan_results.append(ShodanItem(ip=ip, host = host, banner = str(res['banner']),
last_update = str(res['last_update']),
port = str(res['port'])))
except:
pass
print "[+] Shodan results:"
print "==================="
for shodan_result in output_results.shodan_results:
print shodan_result.ip +"(" + shodan_result.port + "): " + \
shodan_result.host + " - updated " + shodan_result.last_update
HarvesterOutput(output_results).process_output(options.output)
def start(argv):
if len(sys.argv) < 4:
usage()
sys.exit()
try:
opts, args = getopt.getopt(argv, "l:d:b:s:vf:nhcte:")
except getopt.GetoptError:
usage()
sys.exit()
host_ip = []
filename = ""
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
limit = 100
dnsserver = False
offset = 0
for opt, arg in opts:
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-s':
offset = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-t':
dnstld = True
elif opt == '-b':
engine = arg
if engine not in ("google", "linkedin", "pgp", "all",
"google-profiles", "bing", "bing_api", "yandex",
"people123", "jigsaw"):
usage()
print("Invalid search engine")
sys.exit()
else:
pass
if engine == "google":
print("[-] Searching in Google:")
search = google.GoogleSearch(word, limit, offset)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
if engine == "exalead":
print("[-] Searching in Exalead:")
search = exaleadsearch.search_exalead(word, limit, offset)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "bing" or engine == "bingapi":
print("[-] Searching in Bing:")
search = bing.BingSearch(word, limit, offset)
if engine == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "yandex": # Not working yet
print("[-] Searching in Yandex:")
search = yandexsearch.search_yandex(word, limit, offset)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "pgp":
print("[-] Searching in PGP key server..")
search = pgpsearch.search_pgp(word)
search.process()
all_emails = search.get_emails()
all_hosts = search.get_hostnames()
elif engine == "people123":
print("[-] Searching in 123People..")
search = people123.search_123people(word, limit)
search.process()
people = search.get_people()
print("Users from 123People:")
print("=====================")
for user in people:
print(user)
sys.exit()
elif engine == "jigsaw":
print("[-] Searching in Jigsaw..")
search = jigsaw.search_jigsaw(word, limit)
search.process()
people = search.get_people()
print("Users from Jigsaw:")
print("=====================")
for user in people:
print(user)
sys.exit()
elif engine == "linkedin":
print("[-] Searching in Linkedin..")
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
print("Users from Linkedin:")
print("====================")
for user in people:
print(user)
sys.exit()
elif engine == "google-profiles":
print("[-] Searching in Google profiles..")
search = google.GoogleSearch(word, limit, offset)
search.process_profiles()
people = search.get_profiles()
print("Users from Google profiles:")
print("---------------------------")
for users in people:
print(users)
sys.exit()
elif engine == "all":
print("Full harvest..")
all_emails = []
all_hosts = []
virtual = "basic"
print("[-] Searching in Google..")
search = google.GoogleSearch(word, limit, offset)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_emails.extend(emails)
all_hosts.extend(hosts)
print("[-] Searching in PGP Key server..")
search = pgp.SearchPGP(word)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print("[-] Searching in Bing..")
bingapi = "no"
search = bing.BingSearch(word, limit, offset)
search.process(bingapi)
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
print("[-] Searching in Exalead..")
search = exalead.ExaLeadSearch(word, limit, offset)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
all_hosts.extend(hosts)
all_emails.extend(emails)
# Results############################################################
print("\n[+] Emails found:")
print("------------------")
if all_emails == []:
print("No emails found")
else:
for emails in all_emails:
print(emails)
print("\n[+] Hosts found in search engines:")
print("------------------------------------")
if all_hosts == []:
print("No hosts found")
else:
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[0]
print(host)
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
# DNS reverse lookup#################################################
dnsrev = []
if dnslookup == True:
print("\n[+] Starting active queries:")
analyzed_ranges = []
for x in full:
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
range = string.join(range, '.')
if not analyzed_ranges.count(range):
print("[-]Performing reverse lookup in :" + range)
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print("Hosts found after reverse lookup:")
print("---------------------------------")
for xh in dnsrev:
print(xh)
# DNS Brute force####################################################
dnsres = []
if dnsbrute == True:
print("[-] Starting DNS brute force:")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print("[+] Hosts found after DNS brute force:\n")
for y in res:
print(y)
dnsres.append(y)
if y not in full:
full.append(y)
#DNS TLD expansion###################################################
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion:")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print("\n[+] Hosts found after DNS TLD expansion:")
print("==========================================")
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
#Virtual hosts search###############################################
if virtual == "basic":
print("[+] Virtual hosts:")
print("==================")
for l in host_ip:
search = bing.BingSearch(l, limit, offset)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
print(l + "\t" + x)
vhost.append(l + ":" + x)
full.append(l + ":" + x)
else:
pass
shodanres = []
shodanvisited = []
if shodan == True:
print("[+] Shodan Database search:")
for x in full:
print(x)
try:
ip = x.split(":")[0]
if not shodanvisited.count(ip):
print("\tSearching for: " + x)
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results:
shodanres.append(x + "SAPO" + str(res['banner']) +
"SAPO" + str(res['port']))
except:
pass
print("[+] Shodan results:")
print("===================")
for x in shodanres:
print(x.split("SAPO")[0] + ":" + x.split("SAPO")[1])
else:
pass
recursion = None
if recursion:
offset = 0
for word in vhost:
search = google.GoogleSearch(word, limit, offset)
search.process()
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
if filename != "":
try:
print("Saving file")
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
sys.exit()
except Exception as e:
print(e)
print("Error creating the file")
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in all_hosts:
file.write('<host>' + x + '</host>')
for x in vhost:
file.write('<vhost>' + x + '</vhost>')
file.write('</theHarvester>')
file.close
def start(argv):
if len(sys.argv) < 4:
Core.usage()
sys.exit(1)
try:
opts, args = getopt.getopt(argv, "l:d:b:s:u:vf:nhcgpte:")
except getopt.GetoptError:
Core.usage()
sys.exit(1)
try:
db = stash.stash_manager()
db.do_init()
except Exception as e:
pass
start = 0
host_ip = []
all_hosts = []
all_emails = []
filename = ""
bingapi = "yes"
dnslookup = False
dnsbrute = False
dnstld = False
shodan = False
vhost = []
virtual = False
ports_scanning = False
takeover_check = False
google_dorking = False
limit = 500
all_ip = []
full = []
trello_info = ([], False)
dnsserver = ""
for value in enumerate(opts):
opt = value[1][0]
arg = value[1][1]
opt = str(opt)
arg = str(arg)
if opt == '-l':
limit = int(arg)
elif opt == '-d':
word = arg
elif opt == '-g':
google_dorking = True
elif opt == '-s':
start = int(arg)
elif opt == '-v':
virtual = "basic"
elif opt == '-f':
filename = arg
elif opt == '-n':
dnslookup = True
elif opt == '-c':
dnsbrute = True
elif opt == '-h':
shodan = True
elif opt == '-e':
dnsserver = arg
elif opt == '-p':
ports_scanning = True
elif opt == '-t':
dnstld = True
elif opt == '-b':
engines = set(arg.split(','))
supportedengines = set([
'baidu', 'bing', 'bingapi', 'censys', 'crtsh', 'cymon',
'dogpile', 'google', 'googleCSE', 'google-certificates',
'google-profiles', 'hunter', 'linkedin', 'netcraft', 'pgp',
'securityTrails', 'threatcrowd', 'trello', 'twitter', 'vhost',
'virustotal', 'yahoo', 'all'
])
if set(engines).issubset(supportedengines):
print(("[-] Target domain: " + word + "\n"))
for engineitem in engines:
if engineitem == "baidu":
print("[-] Searching in Baidu.")
try:
search = baidusearch.search_baidu(word, limit)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'baidu')
db.store_all(word, all_emails, 'email', 'baidu')
except Exception:
pass
elif engineitem == "bing" or engineitem == "bingapi":
print("[-] Searching in Bing.")
try:
search = bingsearch.search_bing(word, limit, start)
if engineitem == "bingapi":
bingapi = "yes"
else:
bingapi = "no"
search.process(bingapi)
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'bing')
db.store_all(word, all_hosts, 'host', 'bing')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "censys":
print("[-] Searching in Censys.")
from discovery import censys
# Import locally or won't work
search = censys.search_censys(word, limit)
search.process()
all_ip = search.get_ipaddresses()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'censys')
db.store_all(word, all_ip, 'ip', 'censys')
elif engineitem == "crtsh":
print("[-] Searching in CRT.sh.")
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
elif engineitem == "cymon":
print("[-] Searching in Cymon.")
from discovery import cymon
# Import locally or won't work.
search = cymon.search_cymon(word)
search.process()
all_ip = search.get_ipaddresses()
db = stash.stash_manager()
db.store_all(word, all_ip, 'ip', 'cymon')
elif engineitem == "dogpile":
print("[-] Searching in Dogpilesearch.")
search = dogpilesearch.search_dogpile(word, limit)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'email', 'dogpile')
db.store_all(word, all_hosts, 'host', 'dogpile')
elif engineitem == "google":
print("[-] Searching in Google.")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
db.store_all(word, all_emails, 'email', 'google')
elif engineitem == "googleCSE":
print("[-] Searching in Google Custom Search.")
try:
search = googleCSE.search_googleCSE(
word, limit, start)
search.process()
search.store_results()
all_emails = filter(search.get_emails())
db = stash.stash_manager()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db.store_all(word, all_hosts, 'email', 'googleCSE')
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'googleCSE')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "google-certificates":
print(
"[-] Searching in Google Certificate transparency report."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
hosts = filter(search.get_domains())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
elif engineitem == "google-profiles":
print("[-] Searching in Google profiles.")
search = googlesearch.search_google(word, limit, start)
search.process_profiles()
people = search.get_profiles()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'google-profile')
print("\nUsers from Google profiles:")
print("---------------------------")
for users in people:
print(users)
sys.exit()
elif engineitem == "hunter":
print("[-] Searching in Hunter.")
from discovery import huntersearch
# Import locally or won't work.
try:
search = huntersearch.search_hunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'hunter')
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "linkedin":
print("[-] Searching in Linkedin.")
search = linkedinsearch.search_linkedin(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'linkedin')
print("\nUsers from Linkedin:")
print("-------------------")
for user in people:
print(user)
sys.exit()
elif engineitem == "netcraft":
print("[-] Searching in Netcraft.")
search = netcraft.search_netcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
elif engineitem == "pgp":
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search.process()
all_emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'pgp')
db.store_all(word, all_emails, 'email', 'pgp')
except Exception:
pass
elif engineitem == 'securityTrails':
print("[-] Searching in SecurityTrails.")
from discovery import securitytrailssearch
try:
search = securitytrailssearch.search_securitytrail(
word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'securityTrails')
ips = search.get_ips()
all_ip.extend(ips)
db = stash.stash_manager()
db.store_all(word, ips, 'ip', 'securityTrails')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
elif engineitem == "threatcrowd":
print("[-] Searching in Threatcrowd.")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
elif engineitem == "trello":
print("[-] Searching in Trello.")
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
elif engineitem == "twitter":
print("[-] Searching in Twitter.")
search = twittersearch.search_twitter(word, limit)
search.process()
people = search.get_people()
db = stash.stash_manager()
db.store_all(word, people, 'name', 'twitter')
print("\nUsers from Twitter:")
print("-------------------")
for user in people:
print(user)
sys.exit()
# vhost
elif engineitem == "virustotal":
print("[-] Searching in VirusTotal.")
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
elif engineitem == "yahoo":
print("[-] Searching in Yahoo.")
search = yahoosearch.search_yahoo(word, limit)
search.process()
hosts = search.get_hostnames()
emails = search.get_emails()
all_hosts.extend(filter(hosts))
all_emails.extend(filter(emails))
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'yahoo')
db.store_all(word, all_emails, 'email', 'yahoo')
elif engineitem == "all":
print(("Full harvest on " + word))
all_emails = []
all_hosts = []
# baidu
print("[-] Searching in Bing.")
bingapi = "no"
search = bingsearch.search_bing(word, limit, start)
search.process(bingapi)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'bing')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'bing')
print("[-] Searching in Censys.")
from discovery import censys
search = censys.search_censys(word, limit)
search.process()
ips = search.get_ipaddresses()
setips = set(ips)
uniqueips = list(setips) # Remove duplicates.
all_ip.extend(uniqueips)
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, uniquehosts, 'host', 'censys')
db.store_all(word, uniqueips, 'ip', 'censys')
print("[-] Searching in CRTSH server.")
search = crtsh.search_crtsh(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'CRTsh')
# cymon
# dogpile
print("[-] Searching in Google.")
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'google')
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'google')
print(
"[-] Searching in Google Certificate transparency report."
)
search = googlecertificates.search_googlecertificates(
word, limit, start)
search.process()
domains = filter(search.get_domains())
all_hosts.extend(domains)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'google-certificates')
# googleplus
# google-certificates
# google-profiles
print("[-] Searching in Hunter.")
from discovery import huntersearch
# Import locally.
try:
search = huntersearch.search_hunter(
word, limit, start)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'hunter')
all_emails.extend(emails)
all_emails = sorted(set(all_emails))
db.store_all(word, all_emails, 'email', 'hunter')
except Exception as e:
if isinstance(e, MissingKey): # Sanity check.
print(e)
else:
pass
# linkedin
print("[-] Searching in Netcraft server.")
search = netcraft.search_netcraft(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'netcraft')
print("[-] Searching in PGP key server.")
try:
search = pgpsearch.search_pgp(word)
search.process()
emails = filter(search.get_emails())
hosts = filter(search.get_hostnames())
sethosts = set(hosts)
uniquehosts = list(sethosts) # Remove duplicates.
all_hosts.extend(uniquehosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'PGP')
all_emails.extend(emails)
db = stash.stash_manager()
db.store_all(word, all_emails, 'email', 'PGP')
except Exception:
pass
print("[-] Searching in ThreatCrowd server.")
try:
search = threatcrowd.search_threatcrowd(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host',
'threatcrowd')
except Exception:
pass
print("[-] Searching in Trello.")
from discovery import trello
# Import locally or won't work.
search = trello.search_trello(word, limit)
search.process()
emails = filter(search.get_emails())
all_emails.extend(emails)
info = search.get_urls()
hosts = filter(info[0])
trello_info = (info[1], True)
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, hosts, 'host', 'trello')
db.store_all(word, emails, 'email', 'trello')
# twitter
# vhost
print("[-] Searching in VirusTotal server.")
search = virustotal.search_virustotal(word)
search.process()
hosts = filter(search.get_hostnames())
all_hosts.extend(hosts)
db = stash.stash_manager()
db.store_all(word, all_hosts, 'host', 'virustotal')
# yahoo
else:
print("[!] Invalid source.\n\n")
sys.exit(1)
# Results
print("\n\033[1;32;40mHarvesting results")
if len(all_ip) == 0:
print("No IP addresses found.")
else:
print("\033[1;33;40m \n[+] IP addresses found:")
print("-----------------------")
print("Total IP addresses: " + str(len(all_ip)) + "\n")
for ip in sorted(list(set(all_ip))):
print(ip)
print("\n[+] Emails found:")
print("-----------------")
# Sanity check to see if all_emails and all_hosts are defined.
try:
all_emails
except NameError:
print('No emails found as all_emails is not defined.')
sys.exit(1)
try:
all_hosts
except NameError:
print('No hosts found as all_hosts is not defined.')
sys.exit(1)
if all_emails == []:
print("No emails found.")
else:
print("Total emails: " + str(len(all_emails)) + "\n")
print(("\n".join(sorted(list(set(all_emails))))))
print("\033[1;33;40m \n[+] Hosts found:")
print("----------------")
if all_hosts == []:
print("No hosts found.")
else:
total = len(all_hosts)
print(("Total hosts: " + str(total) + "\n"))
all_hosts = sorted(list(set(all_hosts)))
for host in all_hosts:
print(host)
print("\033[94m \n[-] Resolving hostnames to IPs.\033[1;33;40m")
print("------------------------------")
full_host = hostchecker.Checker(all_hosts)
full = full_host.check()
for host in full:
ip = host.split(':')[1]
print(host)
if ip != "empty":
if host_ip.count(ip.lower()):
pass
else:
host_ip.append(ip.lower())
db = stash.stash_manager()
db.store_all(word, host_ip, 'ip', 'DNS-resolver')
if trello_info[1] is True: # Indicates user selected Trello.
print("\033[1;33;40m \n[+] URLs found from Trello:")
print("--------------------------")
trello_urls = trello_info[0]
if trello_urls == []:
print('\nNo Trello URLs found.')
else:
total = len(trello_urls)
print(("\nTotal URLs: " + str(total) + "\n"))
for url in sorted(list(set(trello_urls))):
print(url)
# DNS Brute force
dnsres = []
if dnsbrute is True:
print("\n\033[94m[-] Starting DNS brute force. \033[1;33;40m")
a = dnssearch.dns_force(word, dnsserver, verbose=True)
res = a.process()
print("\n\033[94m[-] Hosts found after DNS brute force:")
print("-------------------------------------")
for y in res:
print(y)
dnsres.append(y.split(':')[0])
if y not in full:
full.append(y)
db = stash.stash_manager()
db.store_all(word, dnsres, 'host', 'dns_bruteforce')
# Port Scanning
if ports_scanning is True:
print("\n\n\033[1;32;40m[-] Scanning ports (active).\n")
for x in full:
host = x.split(':')[1]
domain = x.split(':')[0]
if host != "empty":
print(("- Scanning " + host))
ports = [21, 22, 80, 443, 8080]
try:
scan = port_scanner.PortScan(host, ports)
openports = scan.process()
if len(openports) > 1:
print(("\t\033[91m Detected open ports: " +
','.join(str(e)
for e in openports) + "\033[1;32;40m"))
takeover_check = 'True'
if takeover_check == 'True':
if len(openports) > 0:
search_take = takeover.take_over(domain)
search_take.process()
except Exception as e:
print(e)
# DNS reverse lookup
dnsrev = []
if dnslookup is True:
print("\n[+] Starting active queries.")
analyzed_ranges = []
for x in host_ip:
print(x)
ip = x.split(":")[0]
range = ip.split(".")
range[3] = "0/24"
s = '.'
range = s.join(range)
if not analyzed_ranges.count(range):
print(("\033[94m[-] Performing reverse lookup in " + range +
"\033[1;33;40m"))
a = dnssearch.dns_reverse(range, True)
a.list()
res = a.process()
analyzed_ranges.append(range)
else:
continue
for x in res:
if x.count(word):
dnsrev.append(x)
if x not in full:
full.append(x)
print("Hosts found after reverse lookup (in target domain):")
print("----------------------------------------------------")
for xh in dnsrev:
print(xh)
# DNS TLD expansion
dnstldres = []
if dnstld == True:
print("[-] Starting DNS TLD expansion.")
a = dnssearch.dns_tld(word, dnsserver, verbose=True)
res = a.process()
print("\n[+] Hosts found after DNS TLD expansion:")
print("----------------------------------------")
for y in res:
print(y)
dnstldres.append(y)
if y not in full:
full.append(y)
# Virtual hosts search
if virtual == "basic":
print("\n[+] Virtual hosts:")
print("------------------")
for l in host_ip:
search = bingsearch.search_bing(l, limit, start)
search.process_vhost()
res = search.get_allhostnames()
for x in res:
x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
x = re.sub('<', '', x)
x = re.sub('>', '', x)
print((l + "\t" + x))
vhost.append(l + ":" + x)
full.append(l + ":" + x)
vhost = sorted(set(vhost))
else:
pass
# Shodan search
shodanres = []
shodanvisited = []
if shodan is True:
print("\n\n\033[1;32;40m[-] Shodan DB search (passive):\n")
if full == []:
print('No host to search, exiting.')
sys.exit(1)
for x in full:
try:
ip = x.split(":")[1]
if not shodanvisited.count(ip):
print(("\tSearching for: " + ip))
a = shodansearch.search_shodan(ip)
shodanvisited.append(ip)
results = a.run()
for res in results['data']:
shodanres.append(
str("%s:%s - %s - %s - %s," %
(res['ip_str'], res['port'], res['os'],
res['isp'])))
except Exception as e:
pass
print("\n [+] Shodan results:")
print("-------------------")
for x in shodanres:
print(x)
else:
pass
# Here we need to add explosion mode.
# Tengo que sacar los TLD para hacer esto.
recursion = None
if recursion:
start = 0
for word in vhost:
search = googlesearch.search_google(word, limit, start)
search.process(google_dorking)
emails = search.get_emails()
hosts = search.get_hostnames()
print(emails)
print(hosts)
else:
pass
# Reporting
if filename != "":
try:
print("NEW REPORTING BEGINS.")
db = stash.stash_manager()
scanboarddata = db.getscanboarddata()
latestscanresults = db.getlatestscanresults(word)
previousscanresults = db.getlatestscanresults(word,
previousday=True)
latestscanchartdata = db.latestscanchartdata(word)
scanhistorydomain = db.getscanhistorydomain(word)
pluginscanstatistics = db.getpluginscanstatistics()
generator = statichtmlgenerator.htmlgenerator(word)
HTMLcode = generator.beginhtml()
HTMLcode += generator.generatelatestscanresults(latestscanresults)
HTMLcode += generator.generatepreviousscanresults(
previousscanresults)
graph = reportgraph.graphgenerator(word)
HTMLcode += graph.drawlatestscangraph(word, latestscanchartdata)
HTMLcode += graph.drawscattergraphscanhistory(
word, scanhistorydomain)
HTMLcode += generator.generatepluginscanstatistics(
pluginscanstatistics)
HTMLcode += generator.generatedashboardcode(scanboarddata)
HTMLcode += '<p><span style="color: #000000;">Report generated on ' + str(
datetime.datetime.now()) + '</span></p>'
HTMLcode += '''
</body>
</html>
'''
Html_file = open("report.html", "w")
Html_file.write(HTMLcode)
Html_file.close()
print("NEW REPORTING FINISHED!")
print("[+] Saving files.")
html = htmlExport.htmlExport(all_emails, full, vhost, dnsres,
dnsrev, filename, word, shodanres,
dnstldres)
save = html.writehtml()
except Exception as e:
print(e)
print("Error creating the file.")
try:
filename = filename.split(".")[0] + ".xml"
file = open(filename, 'w')
file.write('<?xml version="1.0" encoding="UTF-8"?><theHarvester>')
for x in all_emails:
file.write('<email>' + x + '</email>')
for x in full:
x = x.split(":")
if len(x) == 2:
file.write('<host>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</host>')
else:
file.write('<host>' + x + '</host>')
for x in vhost:
x = x.split(":")
if len(x) == 2:
file.write('<vhost>' + '<ip>' + x[1] + '</ip><hostname>' +
x[0] + '</hostname>' + '</vhost>')
else:
file.write('<vhost>' + x + '</vhost>')
if shodanres != []:
shodanalysis = []
for x in shodanres:
res = x.split("SAPO")
file.write('<shodan>')
file.write('<host>' + res[0] + '</host>')
file.write('<port>' + res[2] + '</port>')
file.write('<banner><!--' + res[1] + '--></banner>')
reg_server = re.compile('Server:.*')
temp = reg_server.findall(res[1])
if temp != []:
shodanalysis.append(res[0] + ":" + temp[0])
file.write('</shodan>')
if shodanalysis != []:
shodanalysis = sorted(set(shodanalysis))
file.write('<servers>')
for x in shodanalysis:
file.write('<server>' + x + '</server>')
file.write('</servers>')
file.write('</theHarvester>')
file.flush()
file.close()
print("Files saved!")
except Exception as er:
print(("Error saving XML file: " + str(er)))
sys.exit()