def is_URL_in_windows(self, main_url):
"""
Detect if platform is Windows or \*NIX. To do this, get the first link, in scope, and
does two resquest. If are the same response, then, platform are Windows. Else are \*NIX.
:returns: True, if the remote host is a Windows system. False is \*NIX or None if unknown.
:rtype: bool
"""
m_forbidden = (
"logout",
"logoff",
"exit",
"sigout",
"signout",
)
# Get the main web page
m_r = download(main_url, callback=self.check_download)
if not m_r or not m_r.raw_data:
return None
discard_data(m_r)
# Get the first link
m_links = None
try:
if m_r.information_type == Information.INFORMATION_HTML:
m_links = extract_from_html(m_r.raw_data, main_url)
else:
m_links = extract_from_text(m_r.raw_data, main_url)
except TypeError,e:
Logger.log_error_more_verbose("Plugin error: %s" % format_exc())
return None
def is_URL_in_windows(self, main_url):
"""
Detect if platform is Windows or \*NIX. To do this, get the first link, in scope, and
does two resquest. If are the same response, then, platform are Windows. Else are \*NIX.
:returns: True, if the remote host is a Windows system. False is \*NIX or None if unknown.
:rtype: bool
"""
m_forbidden = (
"logout",
"logoff",
"exit",
"sigout",
"signout",
)
# Get the main web page
m_r = download(main_url, callback=self.check_download)
if not m_r or not m_r.raw_data:
return None
discard_data(m_r)
# Get the first link
m_links = None
try:
if m_r.information_type == Information.INFORMATION_HTML:
m_links = extract_from_html(m_r.raw_data, main_url)
else:
m_links = extract_from_text(m_r.raw_data, main_url)
except TypeError, e:
Logger.log_error_more_verbose("Plugin error: %s" % format_exc())
return None
def process_url(risk_level, method, matcher, updater_func, total_urls, url):
"""
Checks if an URL exits.
:param risk_level: risk level of the tested URL, if discovered.
:type risk_level: int
:param method: string with HTTP method used.
:type method: str
:param matcher: instance of MatchingAnalyzer object.
:type matcher: `MatchingAnalyzer`
:param updater_func: update_status function to send updates
:type updater_func: update_status
:param total_urls: total number of URL to globally process.
:type total_urls: int
:param url: a tuple with data: (index, the URL to process)
:type url: tuple(int, str)
"""
i, url = url
updater_func((float(i) * 100.0) / float(total_urls))
# Logger.log_more_verbose("Trying to discover URL %s" % url)
# Get URL
p = None
try:
p = HTTP.get_url(url, use_cache=False, method=method)
if p:
discard_data(p)
except Exception, e:
Logger.log_error_more_verbose("Error while processing: '%s': %s" % (url, str(e)))
def __detect_wordpress_installation(self, url, wordpress_urls):
"""
Try to detect a wordpress instalation in the current path.
:param url: URL where try to find the WordPress installation.
:type url: str
:param wordpress_urls: string with wordlist name with WordPress URLs.
:type wordpress_urls: str
:return: True if wordpress installation found. False otherwise.
:rtype: bool
"""
Logger.log_more_verbose(
"Detecting Wordpress instalation in URI: '%s'." % url)
total_urls = 0
urls_found = 0
error_page = get_error_page(url).raw_data
for u in WordListLoader.get_wordlist(wordpress_urls):
total_urls += 1
tmp_url = urljoin(url, u)
r = HTTP.get_url(tmp_url, use_cache=False)
if r.status == "200":
# Try to detect non-default error pages
ratio = get_diff_ratio(r.raw_response, error_page)
if ratio < 0.35:
urls_found += 1
discard_data(r)
# If Oks > 85% continue
if (urls_found / float(total_urls)) < 0.85:
# If all fails, make another last test
url_wp_admin = urljoin(url, "wp-admin/")
try:
p = HTTP.get_url(url_wp_admin,
use_cache=False,
allow_redirects=False)
if p:
discard_data(p)
except Exception, e:
return False
if p.status == "302" and "wp-login.php?redirect_to=" in p.headers.get(
"Location", ""):
return True
else:
return False
def __detect_wordpress_installation(self, url, wordpress_urls):
"""
Try to detect a wordpress instalation in the current path.
:param url: URL where try to find the WordPress installation.
:type url: str
:param wordpress_urls: string with wordlist name with WordPress URLs.
:type wordpress_urls: str
:return: True if wordpress installation found. False otherwise.
:rtype: bool
"""
Logger.log_more_verbose("Detecting Wordpress instalation in URI: '%s'." % url)
total_urls = 0
urls_found = 0
error_page = get_error_page(url).raw_data
for u in WordListLoader.get_wordlist(wordpress_urls):
total_urls += 1
tmp_url = urljoin(url, u)
r = HTTP.get_url(tmp_url, use_cache=False)
if r.status == "200":
# Try to detect non-default error pages
ratio = get_diff_ratio(r.raw_response, error_page)
if ratio < 0.35:
urls_found += 1
discard_data(r)
# If Oks > 85% continue
if (urls_found / float(total_urls)) < 0.85:
# If all fails, make another last test
url_wp_admin = urljoin(url, "wp-admin/")
try:
p = HTTP.get_url(url_wp_admin, use_cache=False, allow_redirects=False)
if p:
discard_data(p)
except Exception, e:
return False
if p.status == "302" and "wp-login.php?redirect_to=" in p.headers.get("Location", ""):
return True
else:
return False
def get_http_method(url):
"""
This function determinates if the method HEAD is available. To do that, compare between two responses:
- One with GET method
- One with HEAD method
If both are seem more than 90%, the response are the same and HEAD method are not allowed.
"""
m_head_response = HTTP.get_url(url, method="HEAD") # FIXME handle exceptions!
discard_data(m_head_response)
m_get_response = HTTP.get_url(url) # FIXME handle exceptions!
discard_data(m_get_response)
# Check if HEAD reponse is different that GET response, to ensure that results are valids
return "HEAD" if HTTP_response_headers_analyzer(m_head_response.headers, m_get_response.headers) < 0.90 else "GET"
def get_http_method(url):
"""
This function determinates if the method HEAD is available. To do that, compare between two responses:
- One with GET method
- One with HEAD method
If both are seem more than 90%, the response are the same and HEAD method are not allowed.
"""
m_head_response = HTTP.get_url(url, method="HEAD") # FIXME handle exceptions!
discard_data(m_head_response)
m_get_response = HTTP.get_url(url) # FIXME handle exceptions!
discard_data(m_get_response)
# Check if HEAD reponse is different that GET response, to ensure that results are valids
return "HEAD" if HTTP_response_headers_analyzer(m_head_response.headers, m_get_response.headers) < 0.90 else "GET"
def get_error_page(url):
"""
Generates an error page an get their content.
:param url: string with the base Url.
:type url: str
:return: a string with the content of response.
:rtype: str
"""
#
# Generate an error in server to get an error page, using a random string
#
# Make the URL
m_error_url = "%s%s" % (url, generate_random_string())
# Get the request
m_error_response = HTTP.get_url(m_error_url) # FIXME handle exceptions!
discard_data(m_error_response)
m_error_response = m_error_response.data
def get_error_page(url):
"""
Generates an error page an get their content.
:param url: string with the base Url.
:type url: str
:return: a string with the content of response.
:rtype: str
"""
#
# Generate an error in server to get an error page, using a random string
#
# Make the URL
m_error_url = "%s%s" % (url, generate_random_string())
# Get the request
m_error_response = HTTP.get_url(m_error_url) # FIXME handle exceptions!
discard_data(m_error_response)
m_error_response = m_error_response.data
def recv_info(self, info):
# Get the response page.
response = HTTP.get_url(info.url, callback=self.check_response)
if response:
try:
# Look for a match.
page_text = response.data
total = float(len(signatures))
for step, (server_name,
server_page) in enumerate(signatures.iteritems()):
# Update status
progress = float(step) / total
self.update_status(progress=progress)
level = get_diff_ratio(page_text, server_page)
if level > 0.95: # magic number :)
# Match found.
vulnerability = DefaultErrorPage(info, server_name)
vulnerability.add_information(response)
return [vulnerability, response]
# Discard the response if no match was found.
discard_data(response)
except Exception:
# Discard the response on error.
discard_data(response)
raise
def recv_info(self, info):
# Get the response page.
response = HTTP.get_url(info.url, callback = self.check_response)
if response:
try:
# Look for a match.
page_text = response.data
total = float(len(signatures))
for step, (server_name, server_page) in enumerate(signatures.iteritems()):
# Update status
progress = float(step) / total
self.update_status(progress=progress)
level = get_diff_ratio(page_text, server_page)
if level > 0.95: # magic number :)
# Match found.
vulnerability = DefaultErrorPage(info, server_name)
vulnerability.add_information(response)
return [vulnerability, response]
# Discard the response if no match was found.
discard_data(response)
except Exception:
# Discard the response on error.
discard_data(response)
raise
def process_url(risk_level, method, matcher, updater_func, total_urls, url):
"""
Checks if an URL exits.
:param risk_level: risk level of the tested URL, if discovered.
:type risk_level: int
:param method: string with HTTP method used.
:type method: str
:param matcher: instance of MatchingAnalyzer object.
:type matcher: `MatchingAnalyzer`
:param updater_func: update_status function to send updates
:type updater_func: update_status
:param total_urls: total number of URL to globally process.
:type total_urls: int
:param url: a tuple with data: (index, the URL to process)
:type url: tuple(int, str)
"""
i, url = url
updater_func((float(i) * 100.0) / float(total_urls))
# Logger.log_more_verbose("Trying to discover URL %s" % url)
# Get URL
p = None
try:
p = HTTP.get_url(url, use_cache=False, method=method)
if p:
discard_data(p)
except Exception, e:
Logger.log_error_more_verbose("Error while processing: '%s': %s" %
(url, str(e)))
class HarvesterPlugin(TestingPlugin):
"""
Integration with `theHarvester <https://code.google.com/p/theharvester/>`_.
"""
# Supported theHarvester modules.
SUPPORTED = (
"google",
"bing",
"pgp",
"exalead", # "yandex",
)
#--------------------------------------------------------------------------
def get_accepted_info(self):
return [Domain]
#--------------------------------------------------------------------------
def recv_info(self, info):
# Get the search parameters.
word = info.hostname
limit = 100
try:
limit = int(Config.plugin_config.get("limit", str(limit)), 0)
except ValueError:
pass
# Search every supported engine.
total = float(len(self.SUPPORTED))
all_emails, all_hosts = set(), set()
for step, engine in enumerate(self.SUPPORTED):
try:
Logger.log_verbose("Searching keyword %r in %s" %
(word, engine))
self.update_status(progress=float(step * 80) / total)
emails, hosts = self.search(engine, word, limit)
except Exception, e:
t = traceback.format_exc()
m = "theHarvester raised an exception: %s\n%s"
warnings.warn(m % (e, t))
continue
all_emails.update(address.lower() for address in emails if address)
all_hosts.update(name.lower() for name in hosts if name)
self.update_status(progress=80)
Logger.log_more_verbose("Search complete for keyword %r" % word)
# Adapt the data into our model.
results = []
# Email addresses.
for address in all_emails:
if "..." in address: # known bug in theHarvester
continue
while address and not address[0].isalnum(
): # known bug in theHarvester
address = address[1:]
while address and not address[-1].isalnum():
address = address[:-1]
if not address:
continue
try:
data = Email(address)
except Exception, e:
warnings.warn("Cannot parse email address: %r" % address)
continue
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
in_scope = data.is_in_scope()
if in_scope:
data.add_resource(info)
results.append(data)
all_hosts.add(data.hostname)
else:
Logger.log_more_verbose("Email address out of scope: %s" %
address)
discard_data(data)
def http_analyzers(main_url, update_status_func, number_of_entries=4):
"""
Analyze HTTP headers for detect the web server. Return a list with most possible web servers.
:param main_url: Base url to test.
:type main_url: str
:param update_status_func: function used to update the status of the process
:type update_status_func: function
:param number_of_entries: number of resutls tu return for most probable web servers detected.
:type number_of_entries: int
:return: Web server family, Web server version, Web server complete description, related web servers (as a dict('SERVER_RELATED' : set(RELATED_NAMES))), others web server with their probabilities as a dict(CONCRETE_WEB_SERVER, PROBABILITY)
"""
# Load wordlist directly related with a HTTP fields.
# { HTTP_HEADER_FIELD : [wordlists] }
m_wordlists_HTTP_fields = {
"Accept-Ranges" : "accept-range",
"Server" : "banner",
"Cache-Control" : "cache-control",
"Connection" : "connection",
"Content-Type" : "content-type",
"WWW-Authenticate" : "htaccess-realm",
"Pragma" : "pragma",
"X-Powered-By" : "x-powered-by"
}
m_actions = {
'GET' : { 'wordlist' : 'Wordlist_get' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/' },
'LONG_GET' : { 'wordlist' : 'Wordlist_get_long' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/%s' % ('a' * 200) },
'NOT_FOUND' : { 'wordlist' : 'Wordlist_get_notfound' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/404_NOFOUND__X02KAS' },
'HEAD' : { 'wordlist' : 'Wordlist_head' , 'weight' : 3 , 'protocol' : 'HTTP/1.1', 'method' : 'HEAD' , 'payload': '/' },
'OPTIONS' : { 'wordlist' : 'Wordlist_options' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'OPTIONS' , 'payload': '/' },
'DELETE' : { 'wordlist' : 'Wordlist_delete' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'DELETE' , 'payload': '/' },
'TEST' : { 'wordlist' : 'Wordlist_attack' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'TEST' , 'payload': '/' },
'INVALID' : { 'wordlist' : 'Wordlist_wrong_method' , 'weight' : 5 , 'protocol' : 'HTTP/9.8', 'method' : 'GET' , 'payload': '/' },
'ATTACK' : { 'wordlist' : 'Wordlist_wrong_version' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': "/etc/passwd?format=%%%%&xss=\x22><script>alert('xss');</script>&traversal=../../&sql='%20OR%201;"}
}
# Store results for others HTTP params
m_d = ParsedURL(main_url)
m_hostname = m_d.hostname
m_port = m_d.port
m_debug = False # Only for develop
# Counter of banners. Used when others methods fails.
m_banners_counter = Counter()
# Score counter
m_counters = HTTPAnalyzer(debug=m_debug)
# Var used to update the status
m_data_len = len(m_actions)
i = 1 # element in process
for l_action, v in m_actions.iteritems():
if m_debug:
print "###########"
l_method = v["method"]
l_payload = v["payload"]
l_proto = v["protocol"]
l_wordlist = v["wordlist"]
# Each type of probe hast different weight.
#
# Weights go from 0 - 5
#
l_weight = v["weight"]
# Make the URL
l_url = urljoin(main_url, l_payload)
# Make the raw request
#l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s:%(port)s\r\nConnection: Close\r\n\r\n" % (
l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s\r\n\r\n" % (
{
"method" : l_method,
"payload" : l_payload,
"protocol" : l_proto,
"host" : m_hostname,
"port" : m_port
}
)
if m_debug:
print "REQUEST"
print l_raw_request
# Do the connection
l_response = None
try:
m_raw_request = HTTP_Raw_Request(l_raw_request)
discard_data(m_raw_request)
l_response = HTTP.make_raw_request(
host = m_hostname,
port = m_port,
raw_request = m_raw_request,
callback = check_raw_response)
if l_response:
discard_data(l_response)
except NetworkException,e:
Logger.log_error_more_verbose("Server-Fingerprint plugin: No response for URL (%s) '%s'. Message: %s" % (l_method, l_url, str(e)))
continue
if not l_response:
Logger.log_error_more_verbose("No response for URL '%s'." % l_url)
continue
if m_debug:
print "RESPONSE"
print l_response.raw_headers
# Update the status
update_status_func((float(i) * 100.0) / float(m_data_len))
Logger.log_more_verbose("Making '%s' test." % (l_wordlist))
i += 1
# Analyze for each wordlist
#
# Store the server banner
try:
m_banners_counter[l_response.headers["Server"]] += l_weight
except KeyError:
pass
#
# =====================
# HTTP directly related
# =====================
#
#
for l_http_header_name, l_header_wordlist in m_wordlists_HTTP_fields.iteritems():
# Check if HTTP header field is in response
if l_http_header_name not in l_response.headers:
continue
l_curr_header_value = l_response.headers[l_http_header_name]
# Generate concrete wordlist name
l_wordlist_path = Config.plugin_extra_config[l_wordlist][l_header_wordlist]
# Load words for the wordlist
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(l_wordlist_path)
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_curr_header_value)
m_counters.inc(l_matches, l_action, l_weight, l_http_header_name, message="HTTP field: " + l_curr_header_value)
#
# =======================
# HTTP INdirectly related
# =======================
#
#
#
# Status code
# ===========
#
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["statuscode"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_response.status)
m_counters.inc(l_matches, l_action, l_weight, "statuscode", message="Status code: " + l_response.status)
#
# Status text
# ===========
#
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["statustext"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_response.reason)
m_counters.inc(l_matches, l_action, l_weight, "statustext", message="Status text: " + l_response.reason)
#
# Header space
# ============
#
# Count the number of spaces between HTTP field name and their value, for example:
# -> Server: Apache 1
# The number of spaces are: 1
#
# -> Server:Apache 1
# The number of spaces are: 0
#
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-space"])
# Looking for matches
try:
l_http_value = l_response.headers[0] # get the value of first HTTP field
l_spaces_num = str(abs(len(l_http_value) - len(l_http_value.lstrip())))
l_matches = l_wordlist_instance.matches_by_value(l_spaces_num)
m_counters.inc(l_matches, l_action, l_weight, "header-space", message="Header space: " + l_spaces_num)
except IndexError:
print "index error header space"
pass
#
# Header capitalafterdash
# =======================
#
# Look for non capitalized first letter of field name, for example:
# -> Content-type: ....
# Instead of:
# -> Content-Type: ....
#
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-capitalafterdash"])
# Looking for matches
l_valid_fields = [x for x in l_response.headers.iterkeys() if "-" in x]
if l_valid_fields:
l_h = l_valid_fields[0]
l_value = l_h.split("-")[1] # Get the second value: Content-type => type
l_dush = None
if l_value[0].isupper(): # Check first letter is lower
l_dush = 1
else:
l_dush = 0
l_matches = l_wordlist_instance.matches_by_value(l_dush)
m_counters.inc(l_matches, l_action, l_weight, "header-capitalizedafterdush", message="Capital after dash: %s" % str(l_dush))
#
# Header order
# ============
#
l_header_order = ','.join(l_response.headers.iterkeys())
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-order"])
l_matches = l_wordlist_instance.matches_by_value(l_header_order)
m_counters.inc(l_matches, l_action, l_weight, "header-order", message="Header order: " + l_header_order)
#
# Protocol name
# ============
#
# For a response like:
# -> HTTP/1.0 200 OK
# ....
#
# Get the 'HTTP' value.
#
try:
l_proto = l_response.protocol # Get the 'HTTP' text from response, if available
if l_proto:
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["protocol-name"])
l_matches = l_wordlist_instance.matches_by_value(l_proto)
m_counters.inc(l_matches, l_action, l_weight, "proto-name", message="Proto name: " + l_proto)
except IndexError:
print "index error protocol name"
pass
#
# Protocol version
# ================
#
# For a response like:
# -> HTTP/1.0 200 OK
# ....
#
# Get the '1.0' value.
#
try:
l_version = l_response.version # Get the '1.0' text from response, if available
if l_version:
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["protocol-version"])
l_matches = l_wordlist_instance.matches_by_value(l_version)
m_counters.inc(l_matches, l_action, l_weight, "proto-version", message="Proto version: " + l_version)
except IndexError:
print "index error protocol version"
pass
if "ETag" in l_response.headers:
l_etag_header = l_response.headers["ETag"]
#
# ETag length
# ================
#
l_etag_len = len(l_etag_header)
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["etag-legth"])
l_matches = l_wordlist_instance.matches_by_value(l_etag_len)
m_counters.inc(l_matches, l_action, l_weight, "etag-length", message="ETag length: " + str(l_etag_len))
#
# ETag Quotes
# ================
#
l_etag_striped = l_etag_header.strip()
if l_etag_striped.startswith("\"") or l_etag_striped.startswith("'"):
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["etag-quotes"])
l_matches = l_wordlist_instance.matches_by_value(l_etag_striped[0])
m_counters.inc(l_matches, l_action, l_weight, "etag-quotes", message="Etag quotes: " + l_etag_striped[0])
if "Vary" in l_response.headers:
l_vary_header = l_response.headers["Vary"]
#
# Vary delimiter
# ================
#
# Checks if Vary header delimiter is something like this:
# -> Vary: Accept-Encoding,User-Agent
# Or this:
# -> Vary: Accept-Encoding, User-Agent
#
l_var_delimiter = ", " if l_vary_header.find(", ") else ","
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-delimiter"])
l_matches = l_wordlist_instance.matches_by_value(l_var_delimiter)
m_counters.inc(l_matches, l_action, l_weight, "vary-delimiter", message="Vary delimiter: " + l_var_delimiter)
#
# Vary capitalizer
# ================
#
# Checks if Vary header delimiter is something like this:
# -> Vary: Accept-Encoding,user-Agent
# Or this:
# -> Vary: accept-encoding,user-agent
#
l_vary_capitalizer = str(0 if l_vary_header == l_vary_header.lower() else 1)
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-capitalize"])
l_matches = l_wordlist_instance.matches_by_value(l_vary_capitalizer)
m_counters.inc(l_matches, l_action, l_weight, "vary-capitalize", message="Vary capitalizer: " + l_vary_capitalizer)
#
# Vary order
# ================
#
# Checks order between vary values:
# -> Vary: Accept-Encoding,user-Agent
# Or this:
# -> Vary: User-Agent,Accept-Encoding
#
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-order"])
l_matches = l_wordlist_instance.matches_by_value(l_vary_header)
m_counters.inc(l_matches, l_action, l_weight, "vary-order", message="Vary order: " + l_vary_header)
#
# =====================
# HTTP specific options
# =====================
#
#
if l_action == "HEAD":
#
# HEAD Options
# ============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-allow", message="HEAD option: " + l_option)
if l_action == "OPTIONS" or l_action == "INVALID" or l_action == "DELETE":
if "Allow" in l_response.headers:
#
# Options allow
# =============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-allow", message="OPTIONS allow: " + l_action + " # " + l_option)
#
# Allow delimiter
# ===============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_var_delimiter = ", " if l_option.find(", ") else ","
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-delimited"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_var_delimiter)
m_counters.inc(l_matches, l_action, l_weight, "options-delimiter", message="OPTION allow delimiter " + l_action + " # " + l_option)
if "Public" in l_response.headers:
#
# Public response
# ===============
#
l_option = l_response.headers.get("Public")
if l_option:
l_wordlist_instance = WordListLoader.get_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-public", message="Public response: " + l_action + " # " + l_option)
def http_simple_analyzer(main_url, update_status_func, number_of_entries=4):
"""Simple method to get fingerprint server info
:param main_url: Base url to test.
:type main_url: str
:param update_status_func: function used to update the status of the process
:type update_status_func: function
:param number_of_entries: number of resutls tu return for most probable web servers detected.
:type number_of_entries: int
:return: a typle as format: Web server family, Web server version, Web server complete description, related web servers (as a dict('SERVER_RELATED' : set(RELATED_NAMES))), others web server with their probabilities as a dict(CONCRETE_WEB_SERVER, PROBABILITY)
"""
m_actions = {
'GET' : { 'wordlist' : 'Wordlist_get' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/' },
'LONG_GET' : { 'wordlist' : 'Wordlist_get_long' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/%s' % ('a' * 200) },
'NOT_FOUND' : { 'wordlist' : 'Wordlist_get_notfound' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/404_NOFOUND__X02KAS' },
'HEAD' : { 'wordlist' : 'Wordlist_head' , 'weight' : 3 , 'protocol' : 'HTTP/1.1', 'method' : 'HEAD' , 'payload': '/' },
'OPTIONS' : { 'wordlist' : 'Wordlist_options' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'OPTIONS' , 'payload': '/' },
'DELETE' : { 'wordlist' : 'Wordlist_delete' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'DELETE' , 'payload': '/' },
'TEST' : { 'wordlist' : 'Wordlist_attack' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'TEST' , 'payload': '/' },
'INVALID' : { 'wordlist' : 'Wordlist_wrong_method' , 'weight' : 5 , 'protocol' : 'HTTP/9.8', 'method' : 'GET' , 'payload': '/' },
'ATTACK' : { 'wordlist' : 'Wordlist_wrong_version' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': "/etc/passwd?format=%%%%&xss=\x22><script>alert('xss');</script>&traversal=../../&sql='%20OR%201;"}
}
m_d = ParsedURL(main_url)
m_hostname = m_d.hostname
m_port = m_d.port
m_scheme = m_d.scheme
m_debug = False # Only for develop
i = 0
m_counters = HTTPAnalyzer()
m_data_len = len(m_actions) # Var used to update the status
m_banners_counter = Counter()
for l_action, v in m_actions.iteritems():
if m_debug:
print "###########"
l_method = v["method"]
l_payload = v["payload"]
l_proto = v["protocol"]
#l_wordlist = v["wordlist"]
# Each type of probe hast different weight.
#
# Weights go from 0 - 5
#
l_weight = v["weight"]
# Make the raw request
l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s\r\n\r\n" % (
{
"method" : l_method,
"payload" : l_payload,
"protocol" : l_proto,
"host" : m_hostname,
"port" : m_port
}
)
if m_debug:
print "REQUEST"
print l_raw_request
# Do the connection
l_response = None
try:
m_raw_request = HTTP_Raw_Request(l_raw_request)
discard_data(m_raw_request)
l_response = HTTP.make_raw_request(
host = m_hostname,
port = m_port,
proto = m_scheme,
raw_request = m_raw_request)
#callback = check_raw_response)
if l_response:
discard_data(l_response)
except NetworkException,e:
Logger.log_error_more_verbose("Server-Fingerprint plugin: No response for host '%s:%d' with method '%s'. Message: %s" % (m_hostname, m_port, l_method, str(e)))
continue
if not l_response:
Logger.log_error_more_verbose("No response for host '%s:%d' with method '%s'." % (m_hostname, m_port, l_method))
continue
if m_debug:
print "RESPONSE"
print l_response.raw_headers
# Update the status
update_status_func((float(i) * 100.0) / float(m_data_len))
Logger.log_more_verbose("Making '%s' test." % l_method)
i += 1
# Analyze for each wordlist
#
# Store the server banner
try:
m_banners_counter[l_response.headers["Server"]] += l_weight
except KeyError:
pass
l_server_name = None
try:
l_server_name = l_response.headers["Server"]
except KeyError:
continue
m_counters.simple_inc(l_server_name, l_method, l_weight)
def run(self, info):
# If it's an URL...
if info.is_instance(BaseURL):
target = URL(info.url)
discard_data(target)
# Get the hostname to test.
hostname = info.hostname
# If it's HTTPS, use the port number from the URL.
if info.is_https:
port = info.parsed_url.port
# Otherwise, assume the port is 443.
else:
port = 443
# Test this port.
is_vulnerable = self.test(hostname, port)
# If it's a service fingerprint...
elif info.is_instance(Relationship(IP, ServiceFingerprint)):
ip, fp = info.instances
target = ip
port = fp.port
starttls = False
# Ignore if the port does not respond directly to SSL...
if fp.protocol != "SSL":
# If it's SMTP, we need to issue a STARTTLS command first.
if fp.name == "smtp":
starttls = True
# Ignore if the port does not support SSL.
else:
Logger.log_more_verbose(
"No SSL services found in fingerprint [%s] for IP %s,"
" aborting." % (fp, ip))
return
# Test this port.
is_vulnerable = self.test(ip.address, port, starttls=starttls)
# Internal error!
else:
assert False, "Unexpected data type received: %s" % type(info)
# If it's vulnerable, report the vulnerability.
if is_vulnerable:
title = "OpenSSL Heartbleed Vulnerability"
description = "An unpatched OpenSSL service was found that's" \
" vulnerable to the Heartbleed vulnerability" \
" (CVE-2014-0162). This vulnerability allows an" \
" attacker to dump the memory contents of the" \
" service running the flawed version of the" \
" OpenSSL library, potentially compromising" \
" usernames, passwords, private keys and other" \
" sensitive data."
references = ["http://heartbleed.com/"]
cve = [
"CVE-2014-0162",
],
if target.is_instance(IP):
vuln = VulnerableService(
target=target,
port=port,
protocol="TCP",
title=title,
description=description,
references=references,
cve=cve,
)
elif target.is_instance(URL):
vuln = VulnerableWebApp(
target=target,
title=title,
description=description,
references=references,
cve=cve,
)
else:
assert False, "Internal error!"
return vuln
def __get_wordpress_version(self, url):
"""
This function get the current version of wordpress and the last version
available for download.
:param url: URL fo target.
:type url: str.
:return: a tuple with (CURRENT_VERSION, LAST_AVAILABLE_VERSION)
:type: tuple(str, str)
"""
url_version = {
# Generic
"wp-login.php": r"(;ver=)([0-9\.]+)([\-a-z]*)",
# For WordPress 3.8
"wp-admin/css/wp-admin-rtl.css": r"(Version[\s]+)([0-9\.]+)",
"wp-admin/css/wp-admin.css": r"(Version[\s]+)([0-9\.]+)"
}
#
# Get current version
#
# URL to find wordpress version
url_current_version = urljoin(url, "readme.html")
current_version_content_1 = download(url_current_version)
if isinstance(current_version_content_1, HTML):
current_version_method1 = re.search(r"(<br/>[\s]*[vV]ersion[\s]*)([0-9\.]*)", current_version_content_1.raw_data)
if current_version_method1 is None:
current_version_method1 = None
else:
if len(current_version_method1.groups()) != 2:
current_version_method1 = None
else:
current_version_method1 = current_version_method1.group(2)
else:
current_version_method1 = None
# Try to find the version into HTML meta value
# Get content of main page
current_version_content_2 = download(url)
# Try to find the info
current_version_method2 = re.search(r"(<meta name=\"generator\" content=\"WordPress[\s]+)([0-9\.]+)",
current_version_content_2.raw_data)
if current_version_method2 is None:
current_version_method2 = None
else:
if len(current_version_method2.groups()) != 2:
current_version_method2 = None
else:
current_version_method2 = current_version_method2.group(2)
# Match versions of the diffentents methods
current_version = "unknown"
if current_version_method1 is None and current_version_method2 is None:
current_version = "unknown"
elif current_version_method1 is None and current_version_method2 is not None:
current_version = current_version_method2
elif current_version_method1 is not None and current_version_method2 is None:
current_version = current_version_method1
elif current_version_method1 is not None and current_version_method2 is not None:
if current_version_method1 != current_version_method2:
current_version = current_version_method2
else:
current_version = current_version_method1
else:
current_version = "unknown"
# If Current version not found
if current_version == "unknown":
for url_pre, regex in url_version.iteritems():
# URL to find wordpress version
url_current_version = urljoin(url, url_pre)
current_version_content = download(url_current_version)
discard_data(current_version_content)
# Find the version
tmp_version = re.search(regex, current_version_content.raw_data)
if tmp_version is not None:
current_version = tmp_version.group(2)
break # Found -> stop search
#
# Get last version
#
# URL to get last version of WordPress available
url_last_version = "http://wordpress.org/download/"
last_version_content = download(url_last_version, allow_out_of_scope=True)
if isinstance(last_version_content, HTML):
last_version = re.search("(WordPress )([0-9\.]*)", last_version_content.raw_data)
if last_version is None:
last_version = "unknown"
else:
if len(last_version.groups()) != 2:
last_version = "unknown"
else:
last_version = last_version.group(2)
else:
last_version = "unknown"
# Discard unused data
discard_data(current_version_content_2)
discard_data(current_version_content_1)
discard_data(last_version_content)
return current_version, last_version
def recv_info(self, info):
m_domain = info.root
# Skips localhost
if m_domain == "localhost":
return
m_return = None
# Checks if the hostname has been already processed
if not self.state.check(m_domain):
#
# Looking for
#
m_subdomains = WordListLoader.get_advanced_wordlist_as_list("subs_small.txt")
# Run in parallel
self.base_domain = m_domain
self.completed = Counter(0)
self.total = len(m_subdomains)
r = pmap(self.get_subdomains_bruteforcer, m_subdomains, pool_size=10)
#
# Remove repeated
#
# The results
m_domains = set()
m_domains_add = m_domains.add
m_domains_already = []
m_domains_already_append = m_domains_already.append
m_ips = set()
m_ips_add = m_ips.add
m_ips_already = []
m_ips_already_append = m_ips_already.append
if r:
for doms in r:
for dom in doms:
# Domains
if dom.type == "CNAME":
if not dom.target in m_domains_already:
m_domains_already_append(dom.target)
if dom.target in Config.audit_scope:
m_domains_add(dom)
else:
discard_data(dom)
# IPs
if dom.type == "A":
if dom.address not in m_ips_already:
m_ips_already_append(dom.address)
m_ips_add(dom)
# Unify
m_domains.update(m_ips)
m_return = m_domains
# Add the information to the host
map(info.add_information, m_return)
# Set the domain as processed
self.state.set(m_domain, True)
Logger.log_verbose("DNS analyzer plugin found %d subdomains" % len(m_return))
# Write the info as more user friendly
if Logger.MORE_VERBOSE:
m_tmp = []
m_tmp_append = m_tmp.append
for x in m_return:
if getattr(x, "address", False):
m_tmp_append("%s (%s)" % (getattr(x, "address"), str(x)))
elif getattr(x, "target", False):
m_tmp_append("%s (%s)" % (getattr(x, "target"), str(x)))
else:
m_tmp_append(str(x))
Logger.log_more_verbose("Subdomains found: \n\t+ %s" % "\n\t+ ".join(m_tmp))
return m_return
def __get_wordpress_version(self, url):
"""
This function get the current version of wordpress and the last version
available for download.
:param url: URL fo target.
:type url: str.
:return: a tuple with (CURRENT_VERSION, LAST_AVAILABLE_VERSION)
:type: tuple(str, str)
"""
url_version = {
# Generic
"wp-login.php": r"(;ver=)([0-9\.]+)([\-a-z]*)",
# For WordPress 3.8
"wp-admin/css/wp-admin-rtl.css": r"(Version[\s]+)([0-9\.]+)",
"wp-admin/css/wp-admin.css": r"(Version[\s]+)([0-9\.]+)"
}
#
# Get current version
#
# URL to find wordpress version
url_current_version = urljoin(url, "readme.html")
current_version_content_1 = download(url_current_version)
if isinstance(current_version_content_1, HTML):
current_version_method1 = re.search(
r"(<br/>[\s]*[vV]ersion[\s]*)([0-9\.]*)",
current_version_content_1.raw_data)
if current_version_method1 is None:
current_version_method1 = None
else:
if len(current_version_method1.groups()) != 2:
current_version_method1 = None
else:
current_version_method1 = current_version_method1.group(2)
else:
current_version_method1 = None
# Try to find the version into HTML meta value
# Get content of main page
current_version_content_2 = download(url)
# Try to find the info
current_version_method2 = re.search(
r"(<meta name=\"generator\" content=\"WordPress[\s]+)([0-9\.]+)",
current_version_content_2.raw_data)
if current_version_method2 is None:
current_version_method2 = None
else:
if len(current_version_method2.groups()) != 2:
current_version_method2 = None
else:
current_version_method2 = current_version_method2.group(2)
# Match versions of the diffentents methods
current_version = "unknown"
if current_version_method1 is None and current_version_method2 is None:
current_version = "unknown"
elif current_version_method1 is None and current_version_method2 is not None:
current_version = current_version_method2
elif current_version_method1 is not None and current_version_method2 is None:
current_version = current_version_method1
elif current_version_method1 is not None and current_version_method2 is not None:
if current_version_method1 != current_version_method2:
current_version = current_version_method2
else:
current_version = current_version_method1
else:
current_version = "unknown"
# If Current version not found
if current_version == "unknown":
for url_pre, regex in url_version.iteritems():
# URL to find wordpress version
url_current_version = urljoin(url, url_pre)
current_version_content = download(url_current_version)
discard_data(current_version_content)
# Find the version
tmp_version = re.search(regex,
current_version_content.raw_data)
if tmp_version is not None:
current_version = tmp_version.group(2)
break # Found -> stop search
#
# Get last version
#
# URL to get last version of WordPress available
url_last_version = "http://wordpress.org/download/"
last_version_content = download(url_last_version,
allow_out_of_scope=True)
if isinstance(last_version_content, HTML):
last_version = re.search("(WordPress )([0-9\.]*)",
last_version_content.raw_data)
if last_version is None:
last_version = "unknown"
else:
if len(last_version.groups()) != 2:
last_version = "unknown"
else:
last_version = last_version.group(2)
else:
last_version = "unknown"
# Discard unused data
discard_data(current_version_content_2)
discard_data(current_version_content_1)
discard_data(last_version_content)
return current_version, last_version
def http_analyzers(main_url, update_status_func, number_of_entries=4):
"""
Analyze HTTP headers for detect the web server. Return a list with most possible web servers.
:param main_url: Base url to test.
:type main_url: str
:param update_status_func: function used to update the status of the process
:type update_status_func: function
:param number_of_entries: number of resutls tu return for most probable web servers detected.
:type number_of_entries: int
:return: Web server family, Web server version, Web server complete description, related web servers (as a dict('SERVER_RELATED' : set(RELATED_NAMES))), others web server with their probabilities as a dict(CONCRETE_WEB_SERVER, PROBABILITY)
"""
# Load wordlist directly related with a HTTP fields.
# { HTTP_HEADER_FIELD : [wordlists] }
m_wordlists_HTTP_fields = {
"Accept-Ranges" : "accept-range",
"Server" : "banner",
"Cache-Control" : "cache-control",
"Connection" : "connection",
"Content-Type" : "content-type",
"WWW-Authenticate" : "htaccess-realm",
"Pragma" : "pragma",
"X-Powered-By" : "x-powered-by"
}
m_actions = {
'GET' : { 'wordlist' : 'Wordlist_get' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/' },
'LONG_GET' : { 'wordlist' : 'Wordlist_get_long' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/%s' % ('a' * 200) },
'NOT_FOUND' : { 'wordlist' : 'Wordlist_get_notfound' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/404_NOFOUND__X02KAS' },
'HEAD' : { 'wordlist' : 'Wordlist_head' , 'weight' : 3 , 'protocol' : 'HTTP/1.1', 'method' : 'HEAD' , 'payload': '/' },
'OPTIONS' : { 'wordlist' : 'Wordlist_options' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'OPTIONS' , 'payload': '/' },
'DELETE' : { 'wordlist' : 'Wordlist_delete' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'DELETE' , 'payload': '/' },
'TEST' : { 'wordlist' : 'Wordlist_attack' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'TEST' , 'payload': '/' },
'INVALID' : { 'wordlist' : 'Wordlist_wrong_method' , 'weight' : 5 , 'protocol' : 'HTTP/9.8', 'method' : 'GET' , 'payload': '/' },
'ATTACK' : { 'wordlist' : 'Wordlist_wrong_version' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': "/etc/passwd?format=%%%%&xss=\x22><script>alert('xss');</script>&traversal=../../&sql='%20OR%201;"}
}
# Store results for others HTTP params
m_d = ParsedURL(main_url)
m_hostname = m_d.hostname
m_port = m_d.port
m_debug = False # Only for develop
# Counter of banners. Used when others methods fails.
m_banners_counter = Counter()
# Score counter
m_counters = HTTPAnalyzer(debug=m_debug)
# Var used to update the status
m_data_len = len(m_actions)
i = 1 # element in process
for l_action, v in m_actions.iteritems():
if m_debug:
print "###########"
l_method = v["method"]
l_payload = v["payload"]
l_proto = v["protocol"]
l_wordlist = v["wordlist"]
# Each type of probe hast different weight.
#
# Weights go from 0 - 5
#
l_weight = v["weight"]
# Make the URL
l_url = urljoin(main_url, l_payload)
# Make the raw request
#l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s:%(port)s\r\nConnection: Close\r\n\r\n" % (
l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s\r\n\r\n" % (
{
"method" : l_method,
"payload" : l_payload,
"protocol" : l_proto,
"host" : m_hostname,
"port" : m_port
}
)
if m_debug:
print "REQUEST"
print l_raw_request
# Do the connection
l_response = None
try:
m_raw_request = HTTP_Raw_Request(l_raw_request)
discard_data(m_raw_request)
l_response = HTTP.make_raw_request(
host = m_hostname,
port = m_port,
raw_request = m_raw_request,
callback = check_raw_response)
if l_response:
discard_data(l_response)
except NetworkException,e:
Logger.log_error_more_verbose("Server-Fingerprint plugin: No response for URL (%s) '%s'. Message: %s" % (l_method, l_url, str(e)))
continue
if not l_response:
Logger.log_error_more_verbose("No response for URL '%s'." % l_url)
continue
if m_debug:
print "RESPONSE"
print l_response.raw_headers
# Update the status
update_status_func((float(i) * 100.0) / float(m_data_len))
Logger.log_more_verbose("Making '%s' test." % (l_wordlist))
i += 1
# Analyze for each wordlist
#
# Store the server banner
try:
m_banners_counter[l_response.headers["Server"]] += l_weight
except KeyError:
pass
#
# =====================
# HTTP directly related
# =====================
#
#
for l_http_header_name, l_header_wordlist in m_wordlists_HTTP_fields.iteritems():
# Check if HTTP header field is in response
if l_http_header_name not in l_response.headers:
continue
l_curr_header_value = l_response.headers[l_http_header_name]
# Generate concrete wordlist name
l_wordlist_path = Config.plugin_extra_config[l_wordlist][l_header_wordlist]
# Load words for the wordlist
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(l_wordlist_path)
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_curr_header_value)
m_counters.inc(l_matches, l_action, l_weight, l_http_header_name, message="HTTP field: " + l_curr_header_value)
#
# =======================
# HTTP INdirectly related
# =======================
#
#
#
# Status code
# ===========
#
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["statuscode"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_response.status)
m_counters.inc(l_matches, l_action, l_weight, "statuscode", message="Status code: " + l_response.status)
#
# Status text
# ===========
#
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["statustext"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_response.reason)
m_counters.inc(l_matches, l_action, l_weight, "statustext", message="Status text: " + l_response.reason)
#
# Header space
# ============
#
# Count the number of spaces between HTTP field name and their value, for example:
# -> Server: Apache 1
# The number of spaces are: 1
#
# -> Server:Apache 1
# The number of spaces are: 0
#
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-space"])
# Looking for matches
try:
l_http_value = l_response.headers[0] # get the value of first HTTP field
l_spaces_num = str(abs(len(l_http_value) - len(l_http_value.lstrip())))
l_matches = l_wordlist_instance.matches_by_value(l_spaces_num)
m_counters.inc(l_matches, l_action, l_weight, "header-space", message="Header space: " + l_spaces_num)
except IndexError:
print "index error header space"
pass
#
# Header capitalafterdash
# =======================
#
# Look for non capitalized first letter of field name, for example:
# -> Content-type: ....
# Instead of:
# -> Content-Type: ....
#
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-capitalafterdash"])
# Looking for matches
l_valid_fields = [x for x in l_response.headers.iterkeys() if "-" in x]
if l_valid_fields:
l_h = l_valid_fields[0]
l_value = l_h.split("-")[1] # Get the second value: Content-type => type
l_dush = None
if l_value[0].isupper(): # Check first letter is lower
l_dush = 1
else:
l_dush = 0
l_matches = l_wordlist_instance.matches_by_value(l_dush)
m_counters.inc(l_matches, l_action, l_weight, "header-capitalizedafterdush", message="Capital after dash: %s" % str(l_dush))
#
# Header order
# ============
#
l_header_order = ','.join(l_response.headers.iterkeys())
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["header-order"])
l_matches = l_wordlist_instance.matches_by_value(l_header_order)
m_counters.inc(l_matches, l_action, l_weight, "header-order", message="Header order: " + l_header_order)
#
# Protocol name
# ============
#
# For a response like:
# -> HTTP/1.0 200 OK
# ....
#
# Get the 'HTTP' value.
#
try:
l_proto = l_response.protocol # Get the 'HTTP' text from response, if available
if l_proto:
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["protocol-name"])
l_matches = l_wordlist_instance.matches_by_value(l_proto)
m_counters.inc(l_matches, l_action, l_weight, "proto-name", message="Proto name: " + l_proto)
except IndexError:
print "index error protocol name"
pass
#
# Protocol version
# ================
#
# For a response like:
# -> HTTP/1.0 200 OK
# ....
#
# Get the '1.0' value.
#
try:
l_version = l_response.version # Get the '1.0' text from response, if available
if l_version:
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["protocol-version"])
l_matches = l_wordlist_instance.matches_by_value(l_version)
m_counters.inc(l_matches, l_action, l_weight, "proto-version", message="Proto version: " + l_version)
except IndexError:
print "index error protocol version"
pass
if "ETag" in l_response.headers:
l_etag_header = l_response.headers["ETag"]
#
# ETag length
# ================
#
l_etag_len = len(l_etag_header)
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["etag-legth"])
l_matches = l_wordlist_instance.matches_by_value(l_etag_len)
m_counters.inc(l_matches, l_action, l_weight, "etag-length", message="ETag length: " + str(l_etag_len))
#
# ETag Quotes
# ================
#
l_etag_striped = l_etag_header.strip()
if l_etag_striped.startswith("\"") or l_etag_striped.startswith("'"):
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["etag-quotes"])
l_matches = l_wordlist_instance.matches_by_value(l_etag_striped[0])
m_counters.inc(l_matches, l_action, l_weight, "etag-quotes", message="Etag quotes: " + l_etag_striped[0])
if "Vary" in l_response.headers:
l_vary_header = l_response.headers["Vary"]
#
# Vary delimiter
# ================
#
# Checks if Vary header delimiter is something like this:
# -> Vary: Accept-Encoding,User-Agent
# Or this:
# -> Vary: Accept-Encoding, User-Agent
#
l_var_delimiter = ", " if l_vary_header.find(", ") else ","
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-delimiter"])
l_matches = l_wordlist_instance.matches_by_value(l_var_delimiter)
m_counters.inc(l_matches, l_action, l_weight, "vary-delimiter", message="Vary delimiter: " + l_var_delimiter)
#
# Vary capitalizer
# ================
#
# Checks if Vary header delimiter is something like this:
# -> Vary: Accept-Encoding,user-Agent
# Or this:
# -> Vary: accept-encoding,user-agent
#
l_vary_capitalizer = str(0 if l_vary_header == l_vary_header.lower() else 1)
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-capitalize"])
l_matches = l_wordlist_instance.matches_by_value(l_vary_capitalizer)
m_counters.inc(l_matches, l_action, l_weight, "vary-capitalize", message="Vary capitalizer: " + l_vary_capitalizer)
#
# Vary order
# ================
#
# Checks order between vary values:
# -> Vary: Accept-Encoding,user-Agent
# Or this:
# -> Vary: User-Agent,Accept-Encoding
#
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["vary-order"])
l_matches = l_wordlist_instance.matches_by_value(l_vary_header)
m_counters.inc(l_matches, l_action, l_weight, "vary-order", message="Vary order: " + l_vary_header)
#
# =====================
# HTTP specific options
# =====================
#
#
if l_action == "HEAD":
#
# HEAD Options
# ============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-allow", message="HEAD option: " + l_option)
if l_action == "OPTIONS" or l_action == "INVALID" or l_action == "DELETE":
if "Allow" in l_response.headers:
#
# Options allow
# =============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-allow", message="OPTIONS allow: " + l_action + " # " + l_option)
#
# Allow delimiter
# ===============
#
l_option = l_response.headers.get("Allow")
if l_option:
l_var_delimiter = ", " if l_option.find(", ") else ","
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-delimited"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_var_delimiter)
m_counters.inc(l_matches, l_action, l_weight, "options-delimiter", message="OPTION allow delimiter " + l_action + " # " + l_option)
if "Public" in l_response.headers:
#
# Public response
# ===============
#
l_option = l_response.headers.get("Public")
if l_option:
l_wordlist_instance = WordListLoader.get_advanced_wordlist_as_dict(Config.plugin_extra_config[l_wordlist]["options-public"])
# Looking for matches
l_matches = l_wordlist_instance.matches_by_value(l_option)
m_counters.inc(l_matches, l_action, l_weight, "options-public", message="Public response: " + l_action + " # " + l_option)
m_analyzer = MatchingAnalyzer(m_response_error_page.data)
m_total = len(m_discovered_urls)
for m_step, l_url in enumerate(m_discovered_urls):
# Update only odd iterations
if m_step % 2:
progress = (float(m_step * 100) / m_total)
self.update_status(progress=progress)
l_url = fix_url(l_url, m_url)
if l_url in Config.audit_scope:
l_p = None
try:
l_p = HTTP.get_url(l_url, callback=self.check_response) # FIXME handle exceptions!
except:
if l_p:
discard_data(l_p)
continue
if l_p:
if m_analyzer.analyze(l_p.data, url=l_url):
match[l_url] = l_p
else:
discard_data(l_p)
# Generate results
for i in m_analyzer.unique_texts:
l_url = i.url
l_p = match.pop(l_url)
m_result = Url(l_url, referer=m_url)
m_result.add_information(l_p)
m_return.append(m_result)
m_return.append(l_p)
def is_URL_in_windows(self, main_url):
"""
Detect if platform is Windows or \*NIX. To do this, get the first link, in scope, and
does two resquest. If are the same response, then, platform are Windows. Else are \*NIX.
:returns: True, if the remote host is a Windows system. False is \*NIX or None if unknown.
:rtype: bool
"""
m_forbidden = (
"logout",
"logoff",
"exit",
"sigout",
"signout",
)
# Get the main web page
m_r = download(main_url, callback=self.check_download)
if not m_r:
return None
discard_data(m_r)
# Get the first link
if m_r.information_type == Information.INFORMATION_HTML:
m_links = extract_from_html(m_r.raw_data, main_url)
else:
m_links = extract_from_text(m_r.raw_data, main_url)
if not m_links:
return None
# Get the first link of the page that's in scope of the audit
m_first_link = None
for u in m_links:
if u in Config.audit_scope and not any(x in u
for x in m_forbidden):
m_first_link = u
break
if not m_first_link:
return None
# Now get two request to the links. One to the original URL and other
# as upper URL.
# Original
m_response_orig = HTTP.get_url(
m_first_link,
callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_orig)
# Uppercase
m_response_upper = HTTP.get_url(
m_first_link.upper(),
callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_upper)
# Compare them
m_orig_data = m_response_orig.raw_response if m_response_orig else ""
m_upper_data = m_response_upper.raw_response if m_response_upper else ""
m_match_level = get_diff_ratio(m_orig_data, m_upper_data)
# If the responses are equal by 90%, two URL are the same => Windows; else => *NIX
m_return = None
if m_match_level > 0.95:
m_return = True
else:
m_return = False
return m_return
def is_URL_in_windows(self, main_url):
"""
Detect if platform is Windows or \*NIX. To do this, get the first link, in scope, and
does two resquest. If are the same response, then, platform are Windows. Else are \*NIX.
:returns: True, if the remote host is a Windows system. False is \*NIX or None if unknown.
:rtype: bool
"""
m_forbidden = (
"logout",
"logoff",
"exit",
"sigout",
"signout",
)
# Get the main web page
m_r = download(main_url, callback=self.check_download)
if not m_r:
return None
discard_data(m_r)
# Get the first link
if m_r.information_type == Information.INFORMATION_HTML:
m_links = extract_from_html(m_r.raw_data, main_url)
else:
m_links = extract_from_text(m_r.raw_data, main_url)
if not m_links:
return None
# Get the first link of the page that's in scope of the audit
m_first_link = None
for u in m_links:
if u in Config.audit_scope and not any(x in u for x in m_forbidden):
m_first_link = u
break
if not m_first_link:
return None
# Now get two request to the links. One to the original URL and other
# as upper URL.
# Original
m_response_orig = HTTP.get_url(m_first_link, callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_orig)
# Uppercase
m_response_upper = HTTP.get_url(m_first_link.upper(), callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_upper)
# Compare them
m_orig_data = m_response_orig.raw_response if m_response_orig else ""
m_upper_data = m_response_upper.raw_response if m_response_upper else ""
m_match_level = get_diff_ratio(m_orig_data, m_upper_data)
# If the responses are equal by 90%, two URL are the same => Windows; else => *NIX
m_return = None
if m_match_level > 0.95:
m_return = True
else:
m_return = False
return m_return
# Check if the url is acceptable by comparing
# the result content.
#
# If the maching level between the error page
# and this url is greater than 52%, then it's
# the same URL and must be discarded.
#
if p and p.status == "200":
# If the method used to get URL was HEAD, get complete URL
if method != "GET":
try:
p = HTTP.get_url(url, use_cache=False, method="GET")
if p:
discard_data(p)
except Exception, e:
Logger.log_more_verbose("Error while processing: '%s': %s" % (url, str(e)))
# Append for analyze and display info if is accepted
if matcher.analyze(p.raw_response, url=url, risk=risk_level):
updater_func(text="Discovered partial url: '%s'" % url)
#----------------------------------------------------------------------
#
# Aux functions
#
#----------------------------------------------------------------------
def load_wordlists(wordlists):
"""
def __find_plugins(self, url, plugins_wordlist, update_func):
"""
Try to find available plugins
:param url: base URL to test.
:type url: str
:param plugins_wordlist: path to wordlist with plugins lists.
:type plugins_wordlist: str
:param update_func: function to update plugin status.
:type update_func: function
:return: list of lists as format:
list([PLUGIN_NAME, PLUGIN_URL, PLUGIN_INSTALLED_VERSION, PLUGIN_LAST_VERSION, [CVE1, CVE2...]])
:type: list(list())
"""
results = []
urls_to_test = {
"readme.txt": r"(Stable tag:[\svV]*)([0-9\.]+)",
"README.txt": r"(Stable tag:[\svV]*)([0-9\.]+)",
}
# Generates the error page
error_response = get_error_page(url).raw_data
# Load plugins info
plugins = []
plugins_append = plugins.append
with open(plugins_wordlist, "rU") as f:
for x in f:
plugins_append(x.replace("\n", ""))
# Calculate sizes
total_plugins = len(plugins)
# Load CSV info
csv_info = csv.reader(plugins)
# Process the URLs
for i, plugin_row in enumerate(csv_info):
# Plugin properties
plugin_URI = plugin_row[0]
plugin_name = plugin_row[1]
plugin_last_version = plugin_row[2]
plugin_CVEs = [] if plugin_row[3] == "" else plugin_row[3].split(
"|")
# Update status
update_func((float(i) * 100.0) / float(total_plugins))
# Make plugin URL
partial_plugin_url = "%s/%s" % (url, "wp-content/plugins/%s" %
plugin_URI)
# Test each URL with possible plugin version info
for target, regex in urls_to_test.iteritems():
plugin_url = "%s/%s" % (partial_plugin_url, target)
# Try to get plugin
p = None
try:
p = HTTP.get_url(plugin_url, use_cache=False)
if p:
discard_data(p)
except Exception, e:
Logger.log_error_more_verbose(
"Error while download: '%s': %s" %
(plugin_url, str(e)))
continue
plugin_installed_version = None
if p.status == "403": # Installed, but inaccesible
plugin_installed_version = "Unknown"
elif p.status == "200":
# Check if page is and non-generic not found page with 404 code
if get_diff_ratio(error_response, p.raw_response) < 0.52:
# Find the version
tmp_version = re.search(regex, p.raw_response)
if tmp_version is not None:
plugin_installed_version = tmp_version.group(2)
# Store info
if plugin_installed_version is not None:
Logger.log(
"Discovered plugin: '%s (installed version: %s)' (latest version: %s)"
% (plugin_name, plugin_installed_version,
plugin_last_version))
results.append([
plugin_name, plugin_url, plugin_installed_version,
plugin_last_version, plugin_CVEs
])
# Plugin found -> not more URL test for this plugin
break
m_total = len(m_discovered_urls)
for m_step, l_url in enumerate(m_discovered_urls):
# Update only odd iterations
if m_step % 2:
progress = (float(m_step * 100) / m_total)
self.update_status(progress=progress)
l_url = fix_url(l_url, m_url)
if l_url in Config.audit_scope:
l_p = None
try:
l_p = HTTP.get_url(l_url, callback=self.check_response
) # FIXME handle exceptions!
except:
if l_p:
discard_data(l_p)
continue
if l_p:
if m_analyzer.analyze(l_p.data, url=l_url):
match[l_url] = l_p
else:
discard_data(l_p)
# Generate results
for i in m_analyzer.unique_texts:
l_url = i.url
l_p = match.pop(l_url)
m_result = URL(l_url, referer=m_url)
m_result.add_information(l_p)
m_return.append(m_result)
m_return.append(l_p)
def run(self, info):
# If it's an URL...
if info.is_instance(BaseURL):
target = URL(info.url)
discard_data(target)
# Get the hostname to test.
hostname = info.hostname
# If it's HTTPS, use the port number from the URL.
if info.is_https:
port = info.parsed_url.port
# Otherwise, assume the port is 443.
else:
port = 443
# Test this port.
is_vulnerable = self.test(hostname, port)
# If it's a service fingerprint...
elif info.is_instance(Relationship(IP, ServiceFingerprint)):
ip, fp = info.instances
target = ip
port = fp.port
starttls = False
# Ignore if the port does not respond directly to SSL...
if fp.protocol != "SSL":
# If it's SMTP, we need to issue a STARTTLS command first.
if fp.name == "smtp":
starttls = True
# Ignore if the port does not support SSL.
else:
Logger.log_more_verbose(
"No SSL services found in fingerprint [%s] for IP %s,"
" aborting." % (fp, ip))
return
# Test this port.
is_vulnerable = self.test(ip.address, port, starttls=starttls)
# Internal error!
else:
assert False, "Unexpected data type received: %s" % type(info)
# If it's vulnerable, report the vulnerability.
if is_vulnerable:
title = "OpenSSL Heartbleed Vulnerability"
description = "An unpatched OpenSSL service was found that's" \
" vulnerable to the Heartbleed vulnerability" \
" (CVE-2014-0162). This vulnerability allows an" \
" attacker to dump the memory contents of the" \
" service running the flawed version of the" \
" OpenSSL library, potentially compromising" \
" usernames, passwords, private keys and other" \
" sensitive data."
references = ["http://heartbleed.com/"]
cve = ["CVE-2014-0162",],
if target.is_instance(IP):
vuln = VulnerableService(
target = target,
port = port,
protocol = "TCP",
title = title,
description = description,
references = references,
cve = cve,
)
elif target.is_instance(URL):
vuln = VulnerableWebApp(
target = target,
title = title,
description = description,
references = references,
cve = cve,
)
else:
assert False, "Internal error!"
return vuln
def http_simple_analyzer(main_url, update_status_func, number_of_entries=4):
"""Simple method to get fingerprint server info
:param main_url: Base url to test.
:type main_url: str
:param update_status_func: function used to update the status of the process
:type update_status_func: function
:param number_of_entries: number of resutls tu return for most probable web servers detected.
:type number_of_entries: int
:return: a typle as format: Web server family, Web server version, Web server complete description, related web servers (as a dict('SERVER_RELATED' : set(RELATED_NAMES))), others web server with their probabilities as a dict(CONCRETE_WEB_SERVER, PROBABILITY)
"""
m_actions = {
'GET' : { 'wordlist' : 'Wordlist_get' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/' },
'LONG_GET' : { 'wordlist' : 'Wordlist_get_long' , 'weight' : 1 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/%s' % ('a' * 200) },
'NOT_FOUND' : { 'wordlist' : 'Wordlist_get_notfound' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': '/404_NOFOUND__X02KAS' },
'HEAD' : { 'wordlist' : 'Wordlist_head' , 'weight' : 3 , 'protocol' : 'HTTP/1.1', 'method' : 'HEAD' , 'payload': '/' },
'OPTIONS' : { 'wordlist' : 'Wordlist_options' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'OPTIONS' , 'payload': '/' },
'DELETE' : { 'wordlist' : 'Wordlist_delete' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'DELETE' , 'payload': '/' },
'TEST' : { 'wordlist' : 'Wordlist_attack' , 'weight' : 5 , 'protocol' : 'HTTP/1.1', 'method' : 'TEST' , 'payload': '/' },
'INVALID' : { 'wordlist' : 'Wordlist_wrong_method' , 'weight' : 5 , 'protocol' : 'HTTP/9.8', 'method' : 'GET' , 'payload': '/' },
'ATTACK' : { 'wordlist' : 'Wordlist_wrong_version' , 'weight' : 2 , 'protocol' : 'HTTP/1.1', 'method' : 'GET' , 'payload': "/etc/passwd?format=%%%%&xss=\x22><script>alert('xss');</script>&traversal=../../&sql='%20OR%201;"}
}
m_d = ParsedURL(main_url)
m_hostname = m_d.hostname
m_port = m_d.port
m_debug = False # Only for develop
i = 0
m_counters = HTTPAnalyzer()
m_data_len = len(m_actions) # Var used to update the status
m_banners_counter = Counter()
for l_action, v in m_actions.iteritems():
if m_debug:
print "###########"
l_method = v["method"]
l_payload = v["payload"]
l_proto = v["protocol"]
#l_wordlist = v["wordlist"]
# Each type of probe hast different weight.
#
# Weights go from 0 - 5
#
l_weight = v["weight"]
# Make the raw request
l_raw_request = "%(method)s %(payload)s %(protocol)s\r\nHost: %(host)s\r\n\r\n" % (
{
"method" : l_method,
"payload" : l_payload,
"protocol" : l_proto,
"host" : m_hostname,
"port" : m_port
}
)
if m_debug:
print "REQUEST"
print l_raw_request
# Do the connection
l_response = None
try:
m_raw_request = HTTP_Raw_Request(l_raw_request)
discard_data(m_raw_request)
l_response = HTTP.make_raw_request(
host = m_hostname,
port = m_port,
raw_request = m_raw_request,
callback = check_raw_response)
if l_response:
discard_data(l_response)
except NetworkException,e:
Logger.log_error_more_verbose("Server-Fingerprint plugin: No response for URL (%s) with method '%s'. Message: %s" % (m_hostname, l_method, str(e)))
continue
if not l_response:
Logger.log_error_more_verbose("No response for host '%s' with method '%s'." % (m_hostname, l_method))
continue
if m_debug:
print "RESPONSE"
print l_response.raw_headers
# Update the status
update_status_func((float(i) * 100.0) / float(m_data_len))
Logger.log_more_verbose("Making '%s' test." % l_method)
i += 1
# Analyze for each wordlist
#
# Store the server banner
try:
m_banners_counter[l_response.headers["Server"]] += l_weight
except KeyError:
pass
l_server_name = None
try:
l_server_name = l_response.headers["Server"]
except KeyError:
continue
m_counters.simple_inc(l_server_name, l_method, l_weight)
class OSFingerprinting(TestingPlugin):
"""
Plugin to fingerprint the remote OS.
"""
#----------------------------------------------------------------------
def get_accepted_info(self):
return [IP, BaseUrl]
#----------------------------------------------------------------------
def recv_info(self, info):
"""
Main function for OS fingerprint. Get a domain or IP and return the fingerprint results.
:param info: Folder URL.
:type info: FolderUrl
:return: OS Fingerprint.
:rtype: OSFingerprint
"""
#
# Detection methods and their weights.
#
# The weight is a value between 1-5
#
FINGERPRINT_METHODS_OS_AND_VERSION = {
'ttl': {
'function': self.ttl_platform_detection,
'weight': 2
}
}
FUNCTIONS = None # Fingerprint methods to run
m_host = None
is_windows = None
if isinstance(info, IP):
m_host = info.address
FUNCTIONS = ['ttl']
else: # BaseUrl
m_host = info.hostname
FUNCTIONS = ['ttl']
# Try to detect if remote system is a Windows
m_windows_host = "%s://%s:%s" % (info.parsed_url.scheme,
info.parsed_url.host,
info.parsed_url.port)
is_windows = self.is_URL_in_windows(m_windows_host)
# Logging
Logger.log_more_verbose(
"Starting OS fingerprinting plugin for site: %s" % m_host)
m_counter = Counter()
# Run functions
for f in FUNCTIONS:
l_function = FINGERPRINT_METHODS_OS_AND_VERSION[f]['function']
### For future use
### l_weight = FINGERPRINT_METHODS_OS_AND_VERSION[f]['weight']
# Run
results = l_function(m_host)
if results:
for l_r in results:
m_counter[l_r] += 1
# Return value
m_return = None
#
# Filter the results
#
if len(m_counter) > 0:
# Fooking for a windows system
if is_windows: # If Windows is detected
l_counter = Counter()
# Extract windows systems
for x, y in m_counter.iteritems():
if "windows" == x:
l_counter[x] += y
# Replace the counter for the new
m_counter = l_counter
# Get most common systems
l_most_common = m_counter.most_common(5)
# First elemente will be the detected OS
m_OS_family = l_most_common[0][0][0]
m_OS_version = l_most_common[0][0][1]
# Next 4 will be the 'others'
m_length = float(len(l_most_common))
m_others = {
"%s-%s" % (l_most_common[i][0][0], l_most_common[i][0][1]):
float('{:.2f}'.format(l_most_common[i][1] / m_length))
for i in xrange(1, len(l_most_common), 1)
}
# create the data
m_return = OSFingerprint(m_OS_family,
m_OS_version,
others=m_others)
elif is_windows is not None:
if is_windows: # Windows system detected
m_return = OSFingerprint("windows")
else: # *NIX system detected
m_return = OSFingerprint("unix_or_compatible")
# If there is information, associate it with the resource
if m_return:
info.add_information(m_return)
return m_return
#----------------------------------------------------------------------
#
# Platform detection methods
#
#----------------------------------------------------------------------
def is_URL_in_windows(self, main_url):
"""
Detect if platform is Windows or \*NIX. To do this, get the first link, in scope, and
does two resquest. If are the same response, then, platform are Windows. Else are \*NIX.
:returns: True, if the remote host is a Windows system. False is \*NIX or None if unknown.
:rtype: bool
"""
m_forbidden = (
"logout",
"logoff",
"exit",
"sigout",
"signout",
)
# Get the main web page
m_r = download(main_url, callback=self.check_download)
if not m_r or not m_r.raw_data:
return None
discard_data(m_r)
# Get the first link
m_links = None
try:
if m_r.information_type == Information.INFORMATION_HTML:
m_links = extract_from_html(m_r.raw_data, main_url)
else:
m_links = extract_from_text(m_r.raw_data, main_url)
except TypeError, e:
Logger.log_error_more_verbose("Plugin error: %s" % format_exc())
return None
if not m_links:
return None
# Get the first link of the page that's in scope of the audit
m_first_link = None
for u in m_links:
if u in Config.audit_scope and not any(x in u
for x in m_forbidden):
m_first_link = u
break
if not m_first_link:
return None
# Now get two request to the links. One to the original URL and other
# as upper URL.
# Original
m_response_orig = HTTP.get_url(
m_first_link,
callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_orig)
# Uppercase
m_response_upper = HTTP.get_url(
m_first_link.upper(),
callback=self.check_response) # FIXME handle exceptions!
discard_data(m_response_upper)
# Compare them
m_orig_data = m_response_orig.raw_response if m_response_orig else ""
m_upper_data = m_response_upper.raw_response if m_response_upper else ""
m_match_level = get_diff_ratio(m_orig_data, m_upper_data)
# If the responses are equal by 90%, two URL are the same => Windows; else => *NIX
m_return = None
if m_match_level > 0.95:
m_return = True
else:
m_return = False
return m_return
# Check if the url is acceptable by comparing
# the result content.
#
# If the maching level between the error page
# and this url is greater than 52%, then it's
# the same URL and must be discarded.
#
if p and p.status == "200":
# If the method used to get URL was HEAD, get complete URL
if method != "GET":
try:
p = HTTP.get_url(url, use_cache=False, method="GET")
if p:
discard_data(p)
except Exception, e:
Logger.log_error_more_verbose(
"Error while processing: '%s': %s" % (url, str(e)))
# Append for analyze and display info if is accepted
if matcher.analyze(p.raw_response, url=url, risk=risk_level):
Logger.log_more_verbose("Discovered partial url: '%s'" % url)
#------------------------------------------------------------------------------
#
# Aux functions
#
#------------------------------------------------------------------------------
def load_wordlists(wordlists):
def __find_plugins(self, url, plugins_wordlist, update_func):
"""
Try to find available plugins
:param url: base URL to test.
:type url: str
:param plugins_wordlist: path to wordlist with plugins lists.
:type plugins_wordlist: str
:param update_func: function to update plugin status.
:type update_func: function
:return: list of lists as format:
list([PLUGIN_NAME, PLUGIN_URL, PLUGIN_INSTALLED_VERSION, PLUGIN_LAST_VERSION, [CVE1, CVE2...]])
:type: list(list())
"""
results = []
urls_to_test = {
"readme.txt": r"(Stable tag:[\svV]*)([0-9\.]+)",
"README.txt": r"(Stable tag:[\svV]*)([0-9\.]+)",
}
# Generates the error page
error_response = get_error_page(url).raw_data
# Load plugins info
plugins = []
plugins_append = plugins.append
with open(plugins_wordlist, "rU") as f:
for x in f:
plugins_append(x.replace("\n", ""))
# Calculate sizes
total_plugins = len(plugins)
# Load CSV info
csv_info = csv.reader(plugins)
# Process the URLs
for i, plugin_row in enumerate(csv_info):
# Plugin properties
plugin_URI = plugin_row[0]
plugin_name = plugin_row[1]
plugin_last_version = plugin_row[2]
plugin_CVEs = [] if plugin_row[3] == "" else plugin_row[3].split("|")
# Update status
update_func((float(i) * 100.0) / float(total_plugins))
# Make plugin URL
partial_plugin_url = "%s/%s" % (url, "wp-content/plugins/%s" % plugin_URI)
# Test each URL with possible plugin version info
for target, regex in urls_to_test.iteritems():
plugin_url = "%s/%s" % (partial_plugin_url, target)
# Try to get plugin
p = None
try:
p = HTTP.get_url(plugin_url, use_cache=False)
if p:
discard_data(p)
except Exception, e:
Logger.log_error_more_verbose("Error while download: '%s': %s" % (plugin_url, str(e)))
continue
plugin_installed_version = None
if p.status == "403": # Installed, but inaccesible
plugin_installed_version = "Unknown"
elif p.status == "200":
# Check if page is and non-generic not found page with 404 code
if get_diff_ratio(error_response, p.raw_response) < 0.52:
# Find the version
tmp_version = re.search(regex, p.raw_response)
if tmp_version is not None:
plugin_installed_version = tmp_version.group(2)
# Store info
if plugin_installed_version is not None:
Logger.log("Discovered plugin: '%s (installed version: %s)' (latest version: %s)" %
(plugin_name, plugin_installed_version, plugin_last_version))
results.append([
plugin_name,
plugin_url,
plugin_installed_version,
plugin_last_version,
plugin_CVEs
])
# Plugin found -> not more URL test for this plugin
break
class ShodanPlugin(TestingPlugin):
"""
This plugin tries to perform passive reconnaissance on a target using
the Shodan web API.
"""
#--------------------------------------------------------------------------
def check_params(self):
# Make sure we have an API key.
self.get_api_key()
#--------------------------------------------------------------------------
def get_accepted_types(self):
return [IP]
#--------------------------------------------------------------------------
def get_api_key(self):
key = Config.plugin_args.get("apikey", None)
if not key:
key = Config.plugin_config.get("apikey", None)
if not key:
raise ValueError(
"Missing API key! Get one at:"
" http://www.shodanhq.com/api_doc")
return key
#--------------------------------------------------------------------------
def run(self, info):
# This is where we'll collect the data we'll return.
results = []
# Skip unsupported IP addresses.
if info.version != 4:
return
ip = info.address
parsed = netaddr.IPAddress(ip)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
# Query Shodan for this host.
try:
key = self.get_api_key()
api = WebAPI(key)
shodan = api.host(ip)
except Exception, e:
tb = traceback.format_exc()
Logger.log_error("Error querying Shodan for host %s: %s" % (ip, str(e)))
Logger.log_error_more_verbose(tb)
return
# Make sure we got the same IP address we asked for.
if ip != shodan.get("ip", ip):
Logger.log_error(
"Shodan gave us a different IP address... weird!")
Logger.log_error_verbose(
"Old IP: %s - New IP: %s" % (ip, shodan["ip"]))
ip = to_utf8( shodan["ip"] )
info = IP(ip)
results.append(info)
# Extract all hostnames and link them to this IP address.
# Note: sometimes Shodan sends IP addresses here! (?)
seen_host = {}
for hostname in shodan.get("hostnames", []):
if hostname == ip:
continue
if hostname in seen_host:
domain = seen_host[hostname]
else:
try:
try:
host = IP(hostname)
except ValueError:
host = Domain(hostname)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
seen_host[hostname] = host
results.append(host)
domain = host
domain.add_resource(info)
# Get the OS fingerprint, if available.
os = to_utf8( shodan.get("os") )
if os:
Logger.log("Host %s is running %s" % (ip, os))
pass # XXX TODO we'll need to reverse lookup the CPE
# Get the GPS data, if available.
# Complete any missing data using the default values.
try:
latitude = float( shodan["latitude"] )
longitude = float( shodan["longitude"] )
except Exception:
latitude = None
longitude = None
if latitude is not None and longitude is not None:
area_code = shodan.get("area_code")
if not area_code:
area_code = None
else:
area_code = str(area_code)
country_code = shodan.get("country_code")
if not country_code:
country_code = shodan.get("country_code3")
if not country_code:
country_code = None
else:
country_code = str(country_code)
else:
country_code = str(country_code)
country_name = shodan.get("country_name")
if not country_name:
country_name = None
city = shodan.get("city")
if not city:
city = None
dma_code = shodan.get("dma_code")
if not dma_code:
dma_code = None
else:
dma_code = str(dma_code)
postal_code = shodan.get("postal_code")
if not postal_code:
postal_code = None
else:
postal_code = str(postal_code)
region_name = shodan.get("region_name")
if not region_name:
region_name = None
geoip = Geolocation(
latitude, longitude,
country_code = country_code,
country_name = country_name,
region_name = region_name,
city = city,
zipcode = postal_code,
metro_code = dma_code,
area_code = area_code,
)
results.append(geoip)
geoip.add_resource(info)
# Go through every result and pick only the latest ones.
latest = {}
for data in shodan.get("data", []):
if (
not "banner" in data or
not "ip" in data or
not "port" in data or
not "timestamp" in data
):
Logger.log_error("Malformed results from Shodan?")
from pprint import pformat
Logger.log_error_more_verbose(pformat(data))
continue
key = (
data["ip"],
data["port"],
data["banner"],
)
try:
timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD)
map(int, data["timestamp"].split(".", 2)))
except Exception:
continue
if key not in latest or timestamp > latest[key][0]:
latest[key] = (timestamp, data)
# Process the latest results.
seen_isp_or_org = set()
seen_html = set()
for _, data in latest.values():
# Extract all domains, but don't link them.
for hostname in data.get("domains", []):
if hostname not in seen_host:
try:
domain = Domain(hostname)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
continue
seen_host[hostname] = domain
results.append(domain)
# We don't have any use for this information yet,
# but log it so at least the user can see it.
isp = to_utf8( data.get("isp") )
org = to_utf8( data.get("org") )
if org and org not in seen_isp_or_org:
seen_isp_or_org.add(org)
Logger.log_verbose(
"Host %s belongs to: %s"
% (ip, org)
)
if isp and (not org or isp != org) and isp not in seen_isp_or_org:
seen_isp_or_org.add(isp)
Logger.log_verbose(
"IP address %s is provided by ISP: %s"
% (ip, isp)
)
# Get the HTML content, if available.
raw_html = to_utf8( data.get("html") )
if raw_html:
hash_raw_html = hash(raw_html)
if hash_raw_html not in seen_html:
seen_html.add(hash_raw_html)
try:
html = HTML(raw_html)
except Exception:
html = None
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
if html:
html.add_resource(info)
results.append(html)
# Get the banner, if available.
raw_banner = to_utf8( data.get("banner") )
try:
port = int( data.get("port", "0") )
except Exception:
port = 0
if raw_banner and port:
try:
banner = Banner(info, raw_banner, port)
except Exception:
banner = None
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
if banner:
results.append(banner)
# Was this host located somewhere else in the past?
for data in reversed(shodan.get("data", [])):
try:
timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD)
map(int, data["timestamp"].split(".", 2)))
old_location = data.get("location")
if old_location:
old_latitude = old_location.get("latitude", latitude)
old_longitude = old_location.get("longitude", longitude)
if (
old_latitude is not None and
old_longitude is not None and
(old_latitude != latitude or old_longitude != longitude)
):
# Get the geoip information.
area_code = old_location.get("area_code")
if not area_code:
area_code = None
country_code = old_location.get("country_code")
if not country_code:
country_code = old_location.get("country_code3")
if not country_code:
country_code = None
country_name = old_location.get("country_name")
if not country_name:
country_name = None
city = old_location.get("city")
if not city:
city = None
postal_code = old_location.get("postal_code")
if not postal_code:
postal_code = None
region_name = old_location.get("region_name")
if not region_name:
region_name = None
geoip = Geolocation(
latitude, longitude,
country_code = country_code,
country_name = country_name,
region_name = region_name,
city = city,
zipcode = postal_code,
area_code = area_code,
)
# If this is the first time we geolocate this IP,
# use this information as it if were up to date.
if latitude is None or longitude is None:
latitude = old_latitude
longitude = old_longitude
results.append(geoip)
geoip.add_resource(info)
# Otherwise, just log the event.
else:
discard_data(geoip)
where = str(geoip)
when = datetime.date(*timestamp)
msg = "Host %s used to be located at %s on %s."
msg %= (ip, where, when.strftime("%B %d, %Y"))
Logger.log_verbose(msg)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
# Return the results.
return results
def recv_info(self, info):
m_domain = info.root
# Skips localhost
if m_domain == "localhost":
return
m_return = None
# Checks if the hostname has been already processed
if not self.state.check(m_domain):
#
# Looking for
#
m_subdomains = WordListLoader.get_advanced_wordlist_as_list(
"subs_small.txt")
# Run in parallel
self.base_domain = m_domain
self.completed = Counter(0)
self.total = len(m_subdomains)
r = pmap(self.get_subdomains_bruteforcer,
m_subdomains,
pool_size=10)
#
# Remove repeated
#
# The results
m_domains = set()
m_domains_add = m_domains.add
m_domains_already = []
m_domains_already_append = m_domains_already.append
m_ips = set()
m_ips_add = m_ips.add
m_ips_already = []
m_ips_already_append = m_ips_already.append
if r:
for doms in r:
for dom in doms:
# Domains
if dom.type == "CNAME":
if not dom.target in m_domains_already:
m_domains_already_append(dom.target)
if dom.target in Config.audit_scope:
m_domains_add(dom)
else:
discard_data(dom)
# IPs
if dom.type == "A":
if dom.address not in m_ips_already:
m_ips_already_append(dom.address)
m_ips_add(dom)
# Unify
m_domains.update(m_ips)
m_return = m_domains
# Add the information to the host
map(info.add_information, m_return)
# Set the domain as processed
self.state.set(m_domain, True)
Logger.log_verbose("DNS analyzer plugin found %d subdomains" %
len(m_return))
# Write the info as more user friendly
if Logger.MORE_VERBOSE:
m_tmp = []
m_tmp_append = m_tmp.append
for x in m_return:
if getattr(x, "address", False):
m_tmp_append("%s (%s)" %
(getattr(x, "address"), str(x)))
elif getattr(x, "target", False):
m_tmp_append("%s (%s)" %
(getattr(x, "target"), str(x)))
else:
m_tmp_append(str(x))
Logger.log_more_verbose("Subdomains found: \n\t+ %s" %
"\n\t+ ".join(m_tmp))
return m_return