class Spider(TestingPlugin):
"""
This plugin is a web spider.
"""
#--------------------------------------------------------------------------
def get_accepted_types(self):
return [URL]
#--------------------------------------------------------------------------
def run(self, info):
m_return = []
m_url = info.url
Logger.log_verbose("Spidering URL: %s" % m_url)
# Check if need follow first redirect, then follow the link.
p = None
try:
allow_redirects = Config.audit_config.follow_redirects or \
(info.depth == 0 and Config.audit_config.follow_first_redirect)
p = download(m_url,
self.check_download,
allow_redirects=allow_redirects)
except NetworkException, e:
Logger.log_error_verbose("Error while processing %r: %s" %
(m_url, str(e)))
if not p:
return m_return
# Send back the data
m_return.append(p)
# TODO: If it's a 301 response, get the Location header
# Get links
m_forms = None
if p.information_type == HTML.data_subtype:
m_links = extract_from_html(p.raw_data, m_url)
m_forms = extract_forms_from_html(p.raw_data, m_url)
#m_links.update( extract_from_text(p.raw_data, m_url) )
elif p.information_type == Text.data_subtype:
m_links = extract_from_text(p.raw_data, m_url)
else:
return m_return
try:
m_links.remove(m_url)
except Exception:
pass
# Do not follow URLs that contain certain keywords
m_forbidden = [
x for x in WordListLoader.get_wordlist_as_list(
Config.plugin_config["wordlist_no_spider"])
]
m_urls_allowed = [
url for url in m_links if not any(x in url for x in m_forbidden)
]
m_urls_not_allowed = m_links.difference(m_urls_allowed)
if m_urls_not_allowed:
Logger.log_more_verbose("Skipped forbidden URLs:\n %s" %
"\n ".join(sorted(m_urls_not_allowed)))
# Do not follow URLs out of scope
m_urls_in_scope = []
m_broken = []
for url in m_urls_allowed:
try:
if url in Config.audit_scope:
m_urls_in_scope.append(url)
except Exception:
m_broken.append(url)
if m_broken:
if len(m_broken) == 1:
Logger.log_more_verbose("Skipped uncrawlable URL: %s" %
m_broken[0])
else:
Logger.log_more_verbose("Skipped uncrawlable URLs:\n %s" %
"\n ".join(sorted(m_broken)))
m_out_of_scope_count = len(m_urls_allowed) - len(
m_urls_in_scope) - len(m_broken)
if m_out_of_scope_count:
Logger.log_more_verbose("Skipped %d links out of scope." %
m_out_of_scope_count)
if m_urls_in_scope:
Logger.log_verbose("Found %d links in URL: %s" %
(len(m_urls_allowed), m_url))
else:
Logger.log_more_verbose("No links found in URL: %s" % m_url)
# Convert to URL data type
for u in m_urls_in_scope:
try:
p = parse_url(u)
if p.scheme == "mailto":
m_resource = Email(p.netloc)
elif p.scheme in ("http", "https"):
m_resource = URL(url=u, referer=m_url)
except Exception:
warn(format_exc(), RuntimeWarning)
print m_resource
m_resource.add_resource(info)
m_return.append(m_resource)
# Get forms info
if m_forms:
m_forms_allowed = [
url for url in m_forms
if not any(x in url[0] for x in m_forbidden)
]
m_forms_not_allowed = {x[0]
for x in m_forms
}.difference(x[0] for x in m_forms_allowed)
else:
m_forms_allowed = []
m_forms_not_allowed = set()
if m_forms_not_allowed:
Logger.log_more_verbose("Skipped forbidden forms:\n %s" %
"\n ".join(sorted(m_forms_not_allowed)))
# Do not follow forms out of scope
m_forms_in_scope = []
m_broken = []
for url in m_forms_allowed:
try:
if url[0] in Config.audit_scope:
m_forms_in_scope.append(url)
except Exception:
m_broken.append(url[0])
if m_broken:
if len(m_broken) == 1:
Logger.log_more_verbose("Skipped uncrawlable forms: %s" %
m_broken[0])
else:
Logger.log_more_verbose("Skipped uncrawlable forms:\n %s" %
"\n ".join(sorted(m_broken)))
m_out_of_scope_count = len(m_forms_allowed) - len(
m_forms_in_scope) - len(m_broken)
if m_out_of_scope_count:
Logger.log_more_verbose("Skipped %d forms out of scope." %
m_out_of_scope_count)
if m_forms_in_scope:
Logger.log_verbose("Found %d forms in URL: %s" %
(len(m_forms_in_scope), m_url))
else:
Logger.log_more_verbose("No forms found in URL: %s" % m_url)
# Convert to URL data type
for u in m_forms_in_scope:
try:
url = u[0]
method = u[1]
params = {x["name"]: x["value"] for x in u[2]}
m_resource = URL(url=url,
referer=m_url,
method=method,
post_params=params)
except Exception:
warn(format_exc(), RuntimeWarning)
m_resource.add_resource(info)
m_return.append(m_resource)
# Send the results
return m_return
def sf_EMAILADDR(self, sf_module, source, raw_data):
return Email(raw_data)
class HarvesterPlugin(TestingPlugin):
"""
Integration with `theHarvester <https://code.google.com/p/theharvester/>`_.
"""
# Supported theHarvester modules.
SUPPORTED = (
"google",
"bing",
"pgp",
"exalead", # "yandex",
)
#--------------------------------------------------------------------------
def get_accepted_info(self):
return [Domain]
#--------------------------------------------------------------------------
def recv_info(self, info):
# Get the search parameters.
word = info.hostname
limit = 100
try:
limit = int(Config.plugin_config.get("limit", str(limit)), 0)
except ValueError:
pass
# Search every supported engine.
total = float(len(self.SUPPORTED))
all_emails, all_hosts = set(), set()
for step, engine in enumerate(self.SUPPORTED):
try:
Logger.log_verbose("Searching keyword %r in %s" %
(word, engine))
self.update_status(progress=float(step * 80) / total)
emails, hosts = self.search(engine, word, limit)
except Exception, e:
t = traceback.format_exc()
m = "theHarvester raised an exception: %s\n%s"
warnings.warn(m % (e, t))
continue
all_emails.update(address.lower() for address in emails if address)
all_hosts.update(name.lower() for name in hosts if name)
self.update_status(progress=80)
Logger.log_more_verbose("Search complete for keyword %r" % word)
# Adapt the data into our model.
results = []
# Email addresses.
for address in all_emails:
if "..." in address: # known bug in theHarvester
continue
while address and not address[0].isalnum(
): # known bug in theHarvester
address = address[1:]
while address and not address[-1].isalnum():
address = address[:-1]
if not address:
continue
try:
data = Email(address)
except Exception, e:
warnings.warn("Cannot parse email address: %r" % address)
continue
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
in_scope = data.is_in_scope()
if in_scope:
data.add_resource(info)
results.append(data)
all_hosts.add(data.hostname)
else:
Logger.log_more_verbose("Email address out of scope: %s" %
address)
discard_data(data)