def test_remove_table(self): disk_dict = DiskDict() table_name = disk_dict.table_name db = get_default_temp_db_instance() self.assertTrue(db.table_exists(table_name)) disk_dict.cleanup() self.assertFalse(db.table_exists(table_name))
def test_table_with_prefix(self): _unittest = 'unittest' disk_dict = DiskDict(_unittest) self.assertIn(_unittest, disk_dict.table_name) db = get_default_temp_db_instance() self.assertTrue(db.table_exists(disk_dict.table_name)) disk_dict.cleanup() self.assertFalse(db.table_exists(disk_dict.table_name))
class html_comments(GrepPlugin): """ Extract and analyze HTML comments. :author: Andres Riancho ([email protected]) """ HTML_RE = re.compile('<[a-zA-Z]*.*?>.*?</[a-zA-Z]>') INTERESTING_WORDS = ( # In English 'user', 'pass', 'xxx', 'fix', 'bug', 'broken', 'oops', 'hack', 'caution', 'todo', 'note', 'warning', '!!!', '???', 'shit', 'pass', 'password', 'passwd', 'pwd', 'secret', 'stupid', # In Spanish 'tonto', 'porqueria', 'cuidado', 'usuario', u'contraseña', 'puta', 'email', 'security', 'captcha', 'pinga', 'cojones', # some in Portuguese 'banco', 'bradesco', 'itau', 'visa', 'bancoreal', u'transfêrencia', u'depósito', u'cartão', u'crédito', 'dados pessoais') _multi_in = multi_in([' %s ' % w for w in INTERESTING_WORDS]) def __init__(self): GrepPlugin.__init__(self) # Internal variables self._comments = DiskDict(table_prefix='html_comments') self._already_reported = ScalableBloomFilter() def grep(self, request, response): """ Plugin entry point, parse those comments! :param request: The HTTP request object. :param response: The HTTP response object :return: None """ if not response.is_text_or_html(): return try: dp = parser_cache.dpc.get_document_parser_for(response) except BaseFrameworkException: return for comment in dp.get_comments(): # These next two lines fix this issue: # audit.ssi + grep.html_comments + web app with XSS = false positive if request.sent(comment): continue if self._is_new(comment, response): self._interesting_word(comment, request, response) self._html_in_comment(comment, request, response) def _interesting_word(self, comment, request, response): """ Find interesting words in HTML comments """ comment = comment.lower() for word in self._multi_in.query(comment): if (word, response.get_url()) in self._already_reported: continue desc = ('A comment with the string "%s" was found in: "%s".' ' This could be interesting.') desc %= (word, response.get_url()) v = Vuln.from_fr('Interesting HTML comment', desc, severity.INFORMATION, response.id, self.get_name(), request) v.add_to_highlight(word) kb.kb.append(self, 'interesting_comments', v) self._already_reported.add((word, response.get_url())) def _html_in_comment(self, comment, request, response): """ Find HTML code in HTML comments """ html_in_comment = self.HTML_RE.search(comment) if html_in_comment is None: return if (comment, response.get_url()) in self._already_reported: return # There is HTML code in the comment. comment = comment.strip() comment = comment.replace('\n', '') comment = comment.replace('\r', '') comment = comment[:40] desc = ('A comment with the string "%s" was found in: "%s".' ' This could be interesting.') desc %= (comment, response.get_url()) v = Vuln.from_fr('HTML comment contains HTML code', desc, severity.INFORMATION, response.id, self.get_name(), request) v.set_uri(response.get_uri()) v.add_to_highlight(html_in_comment.group(0)) om.out.vulnerability(v.get_desc(), severity=severity.INFORMATION) kb.kb.append(self, 'html_comment_hides_html', v) self._already_reported.add((comment, response.get_url())) def _is_new(self, comment, response): """ Make sure that we perform a thread safe check on the self._comments dict, in order to avoid duplicates. """ with self._plugin_lock: #pylint: disable=E1103 comment_data = self._comments.get(comment, None) response_url = response.get_url() if comment_data is None: self._comments[comment] = [(response_url, response.id)] return True else: for saved_url, response_id in comment_data: if response_url == saved_url: return False else: comment_data.append((response_url, response.id)) self._comments[comment] = comment_data return True #pylint: enable=E1103 def end(self): """ This method is called when the plugin wont be used anymore. :return: None """ for comment, url_request_id_lst in self._comments.iteritems(): stick_comment = ' '.join(comment.split()) if len(stick_comment) > 40: msg = ('A comment with the string "%s..." (and %s more bytes)' ' was found on these URL(s):') args = (stick_comment[:40], str(len(stick_comment) - 40)) om.out.vulnerability(msg % args, severity=severity.INFORMATION) else: msg = 'A comment containing "%s" was found on these URL(s):' om.out.vulnerability(msg % stick_comment, severity=severity.INFORMATION) inform = [] for url, request_id in url_request_id_lst: msg = '- %s (request with id: %s)' inform.append(msg % (url, request_id)) for i in sorted(inform): om.out.vulnerability(i, severity=severity.INFORMATION) self._comments.cleanup() def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class VariantDB(object): """ See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also understand that we'll keep "dirty" versions of the references/fuzzable requests in order to be able to answer "False" to a call for need_more_variants in a situation like this: >> need_more_variants('http://foo.com/abc?id=32') True >> append('http://foo.com/abc?id=32') True >> need_more_variants('http://foo.com/abc?id=32') False """ HASH_IGNORE_HEADERS = ('referer', ) TAG = '[variant_db]' def __init__(self): self._variants = DiskDict(table_prefix='variant_db') self._variants_eq = ScalableBloomFilter() self._variants_form = DiskDict(table_prefix='variant_db_form') self.params_max_variants = cf.cf.get('params_max_variants') self.path_max_variants = cf.cf.get('path_max_variants') self.max_equal_form_variants = cf.cf.get('max_equal_form_variants') self._db_lock = threading.RLock() def cleanup(self): self._variants.cleanup() self._variants_form.cleanup() def append(self, fuzzable_request): """ :return: True if we added a new fuzzable request variant to the DB, False if NO more variants are required for this fuzzable request. """ with self._db_lock: if self._seen_exactly_the_same(fuzzable_request): return False if self._has_form(fuzzable_request): if not self._need_more_variants_for_form(fuzzable_request): return False if not self._need_more_variants_for_uri(fuzzable_request): return False # Yes, please give me more variants of fuzzable_request return True def _log_return_false(self, fuzzable_request, reason): args = (reason, fuzzable_request) msg = 'VariantDB is returning False because of "%s" for "%s"' om.out.debug(msg % args) def _need_more_variants_for_uri(self, fuzzable_request): # # Do we need more variants for the fuzzable request? (similar match) # PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS # clean_dict_key = clean_fuzzable_request(fuzzable_request) count = self._variants.get(clean_dict_key, None) if count is None: self._variants[clean_dict_key] = 1 return True # We've seen at least one fuzzable request with this pattern... url = fuzzable_request.get_uri() has_params = url.has_query_string() or fuzzable_request.get_raw_data() # Choose which max_variants to use if has_params: max_variants = self.params_max_variants max_variants_type = 'params' else: max_variants = self.path_max_variants max_variants_type = 'path' if count >= max_variants: _type = 'need_more_variants_for_uri(%s)' % max_variants_type self._log_return_false(fuzzable_request, _type) return False self._variants[clean_dict_key] = count + 1 return True def _seen_exactly_the_same(self, fuzzable_request): # # Is the fuzzable request already known to us? (exactly the same) # request_hash = fuzzable_request.get_request_hash( self.HASH_IGNORE_HEADERS) if request_hash in self._variants_eq: return True # Store it to avoid duplicated fuzzable requests in our framework self._variants_eq.add(request_hash) self._log_return_false(fuzzable_request, 'seen_exactly_the_same') return False def _has_form(self, fuzzable_request): raw_data = fuzzable_request.get_raw_data() if raw_data and len(raw_data.get_param_names()) >= 2: return True return False def _need_more_variants_for_form(self, fuzzable_request): # # Do we need more variants for this form? (similar match) # MAX_EQUAL_FORM_VARIANTS # clean_dict_key_form = clean_fuzzable_request_form(fuzzable_request) count = self._variants_form.get(clean_dict_key_form, None) if count is None: self._variants_form[clean_dict_key_form] = 1 return True if count >= self.max_equal_form_variants: self._log_return_false(fuzzable_request, 'need_more_variants_for_form') return False self._variants_form[clean_dict_key_form] = count + 1 return True
class VariantDB(object): """ See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also understand that we'll keep "dirty" versions of the references/fuzzable requests in order to be able to answer "False" to a call for need_more_variants in a situation like this: need_more_variants('http://foo.com/abc?id=32') --> True append('http://foo.com/abc?id=32') need_more_variants('http://foo.com/abc?id=32') --> False """ HASH_IGNORE_HEADERS = ('referer', ) TAG = '[variant_db]' def __init__(self, params_max_variants=PARAMS_MAX_VARIANTS, path_max_variants=PATH_MAX_VARIANTS): self._variants_eq = DiskDict(table_prefix='variant_db_eq') self._variants = DiskDict(table_prefix='variant_db') self.params_max_variants = params_max_variants self.path_max_variants = path_max_variants self._db_lock = threading.RLock() def cleanup(self): self._variants_eq.cleanup() self._variants.cleanup() def append(self, fuzzable_request): """ :return: True if we added a new fuzzable request variant to the DB, False if no more variants are required for this fuzzable request. """ with self._db_lock: # # Is the fuzzable request already known to us? (exactly the same) # request_hash = fuzzable_request.get_request_hash( self.HASH_IGNORE_HEADERS) already_seen = self._variants_eq.get(request_hash, False) if already_seen: return False # Store it to avoid duplicated fuzzable requests in our framework self._variants_eq[request_hash] = True # # Do we need more variants for the fuzzable request? (similar match) # clean_dict_key = clean_fuzzable_request(fuzzable_request) count = self._variants.get(clean_dict_key, None) if count is None: self._variants[clean_dict_key] = 1 return True # We've seen at least one fuzzable request with this pattern... url = fuzzable_request.get_uri() has_params = url.has_query_string( ) or fuzzable_request.get_raw_data() # Choose which max_variants to use if has_params: max_variants = self.params_max_variants else: max_variants = self.path_max_variants if count >= max_variants: return False else: self._variants[clean_dict_key] = count + 1 return True
class ssi(AuditPlugin): """ Find server side inclusion vulnerabilities. :author: Andres Riancho ([email protected]) """ def __init__(self): AuditPlugin.__init__(self) # Internal variables self._expected_res_mutant = DiskDict() self._freq_list = DiskList() re_str = '<!--#exec cmd="echo -n (.*?);echo -n (.*?)" -->' self._extract_results_re = re.compile(re_str) def audit(self, freq, orig_response): """ Tests an URL for server side inclusion vulnerabilities. :param freq: A FuzzableRequest """ # Create the mutants to send right now, ssi_strings = self._get_ssi_strings() mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response) # Used in end() to detect "persistent SSI" for mut in mutants: expected_result = self._extract_result_from_payload( mut.get_token_value()) self._expected_res_mutant[expected_result] = mut self._freq_list.append(freq) # End of persistent SSI setup self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants, self._analyze_result) def _get_ssi_strings(self): """ This method returns a list of server sides to try to include. :return: A string, see above. """ yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % (rand_alpha(5), rand_alpha(5)) # TODO: Add mod_perl ssi injection support # http://www.sens.buffalo.edu/services/webhosting/advanced/perlssi.shtml #yield <!--#perl sub="sub {print qq/If you see this, mod_perl is working!/;}" --> def _extract_result_from_payload(self, payload): """ Extract the expected result from the payload we're sending. """ match = self._extract_results_re.search(payload) return match.group(1) + match.group(2) def _analyze_result(self, mutant, response): """ Analyze the result of the previously sent request. :return: None, save the vuln to the kb. """ if self._has_no_bug(mutant): e_res = self._extract_result_from_payload(mutant.get_token_value()) if e_res in response and not e_res in mutant.get_original_response_body(): desc = 'Server side include (SSI) was found at: %s' desc = desc % mutant.found_at() v = Vuln.from_mutant('Server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(e_res) self.kb_append_uniq(self, 'ssi', v) def end(self): """ This method is called when the plugin wont be used anymore and is used to find persistent SSI vulnerabilities. Example where a persistent SSI can be found: Say you have a "guestbook" (a CGI application that allows visitors to leave messages for everyone to see) on a server that has SSI enabled. Most such guestbooks around the Net actually allow visitors to enter HTML code as part of their comments. Now, what happens if a malicious visitor decides to do some damage by entering the following: <!--#exec cmd="ls" --> If the guestbook CGI program was designed carefully, to strip SSI commands from the input, then there is no problem. But, if it was not, there exists the potential for a major headache! For a working example please see moth VM. """ multi_in_inst = multi_in(self._expected_res_mutant.keys()) def filtered_freq_generator(freq_list): already_tested = ScalableBloomFilter() for freq in freq_list: if freq not in already_tested: already_tested.add(freq) yield freq def analyze_persistent(freq, response): for matched_expected_result in multi_in_inst.query(response.get_body()): # We found one of the expected results, now we search the # self._persistent_data to find which of the mutants sent it # and create the vulnerability mutant = self._expected_res_mutant[matched_expected_result] desc = 'Server side include (SSI) was found at: %s' \ ' The result of that injection is shown by browsing'\ ' to "%s".' desc = desc % (mutant.found_at(), freq.get_url()) v = Vuln.from_mutant('Persistent server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(matched_expected_result) self.kb_append(self, 'ssi', v) self._send_mutants_in_threads(self._uri_opener.send_mutant, filtered_freq_generator(self._freq_list), analyze_persistent, cache=False) self._expected_res_mutant.cleanup() self._freq_list.cleanup() def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class ssi(AuditPlugin): """ Find server side inclusion vulnerabilities. :author: Andres Riancho ([email protected]) """ def __init__(self): AuditPlugin.__init__(self) # Internal variables self._expected_mutant_dict = DiskDict(table_prefix="ssi") self._extract_expected_re = re.compile("[1-9]{5}") def audit(self, freq, orig_response): """ Tests an URL for server side inclusion vulnerabilities. :param freq: A FuzzableRequest """ ssi_strings = self._get_ssi_strings() mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response) self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants, self._analyze_result) def _get_ssi_strings(self): """ This method returns a list of server sides to try to include. :return: A string, see above. """ # Generic yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % get_seeds() # Perl SSI yield ( '<!--#set var="SEED_A" value="%s" -->' '<!--#echo var="SEED_A" -->' '<!--#set var="SEED_B" value="%s" -->' '<!--#echo var="SEED_B" -->' % get_seeds() ) # Smarty # http://www.smarty.net/docsv2/en/language.function.math.tpl yield '{math equation="x * y" x=%s y=%s}' % get_seeds() # Mako # http://docs.makotemplates.org/en/latest/syntax.html yield "${%s * %s}" % get_seeds() # Jinja2 and Twig # http://jinja.pocoo.org/docs/dev/templates/#math # http://twig.sensiolabs.org/doc/templates.html yield "{{%s * %s}}" % get_seeds() # Generic yield "{%s * %s}" % get_seeds() def _get_expected_results(self, mutant): """ Extracts the potential results from the mutant payload and returns them in a list. """ sent_payload = mutant.get_token_payload() seed_numbers = self._extract_expected_re.findall(sent_payload) seed_a = int(seed_numbers[0]) seed_b = int(seed_numbers[1]) return [str(seed_a * seed_b), "%s%s" % (seed_a, seed_b)] def _analyze_result(self, mutant, response): """ Analyze the result of the previously sent request. :return: None, save the vuln to the kb. """ # Store the mutants in order to be able to analyze the persistent case # later expected_results = self._get_expected_results(mutant) for expected_result in expected_results: self._expected_mutant_dict[expected_result] = mutant # Now we analyze the "reflected" case if self._has_bug(mutant): return for expected_result in expected_results: if expected_result not in response: continue if expected_result in mutant.get_original_response_body(): continue desc = "Server side include (SSI) was found at: %s" desc %= mutant.found_at() v = Vuln.from_mutant( "Server side include vulnerability", desc, severity.HIGH, response.id, self.get_name(), mutant ) v.add_to_highlight(expected_result) self.kb_append_uniq(self, "ssi", v) def end(self): """ This method is called when the plugin wont be used anymore and is used to find persistent SSI vulnerabilities. Example where a persistent SSI can be found: Say you have a "guest book" (a CGI application that allows visitors to leave messages for everyone to see) on a server that has SSI enabled. Most such guest books around the Net actually allow visitors to enter HTML code as part of their comments. Now, what happens if a malicious visitor decides to do some damage by entering the following: <!--#exec cmd="ls" --> If the guest book CGI program was designed carefully, to strip SSI commands from the input, then there is no problem. But, if it was not, there exists the potential for a major headache! For a working example please see moth VM. """ fuzzable_request_set = kb.kb.get_all_known_fuzzable_requests() self._send_mutants_in_threads( self._uri_opener.send_mutant, fuzzable_request_set, self._analyze_persistent, cache=False ) self._expected_mutant_dict.cleanup() def _analyze_persistent(self, freq, response): """ Analyze the response of sending each fuzzable request found by the framework, trying to identify any locations where we might have injected a payload. :param freq: The fuzzable request :param response: The HTTP response :return: None, vulns are stored in KB """ multi_in_inst = multi_in(self._expected_mutant_dict.keys()) for matched_expected_result in multi_in_inst.query(response.get_body()): # We found one of the expected results, now we search the # self._expected_mutant_dict to find which of the mutants sent it # and create the vulnerability mutant = self._expected_mutant_dict[matched_expected_result] desc = ( "Server side include (SSI) was found at: %s" " The result of that injection is shown by browsing" ' to "%s".' ) desc %= (mutant.found_at(), freq.get_url()) v = Vuln.from_mutant( "Persistent server side include vulnerability", desc, severity.HIGH, response.id, self.get_name(), mutant, ) v.add_to_highlight(matched_expected_result) self.kb_append(self, "ssi", v) def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class VariantDB(object): """ See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also understand that we'll keep "dirty" versions of the references/fuzzable requests in order to be able to answer "False" to a call for need_more_variants in a situation like this: need_more_variants('http://foo.com/abc?id=32') --> True append('http://foo.com/abc?id=32') need_more_variants('http://foo.com/abc?id=32') --> False """ HASH_IGNORE_HEADERS = ('referer',) TAG = '[variant_db]' def __init__(self, params_max_variants=PARAMS_MAX_VARIANTS, path_max_variants=PATH_MAX_VARIANTS): self._variants_eq = DiskDict(table_prefix='variant_db_eq') self._variants = DiskDict(table_prefix='variant_db') self.params_max_variants = params_max_variants self.path_max_variants = path_max_variants self._db_lock = threading.RLock() def cleanup(self): self._variants_eq.cleanup() self._variants.cleanup() def append(self, fuzzable_request): """ :return: True if we added a new fuzzable request variant to the DB, False if no more variants are required for this fuzzable request. """ with self._db_lock: # # Is the fuzzable request already known to us? (exactly the same) # request_hash = fuzzable_request.get_request_hash(self.HASH_IGNORE_HEADERS) already_seen = self._variants_eq.get(request_hash, False) if already_seen: return False # Store it to avoid duplicated fuzzable requests in our framework self._variants_eq[request_hash] = True # # Do we need more variants for the fuzzable request? (similar match) # clean_dict_key = clean_fuzzable_request(fuzzable_request) count = self._variants.get(clean_dict_key, None) if count is None: self._variants[clean_dict_key] = 1 return True # We've seen at least one fuzzable request with this pattern... url = fuzzable_request.get_uri() has_params = url.has_query_string() or fuzzable_request.get_raw_data() # Choose which max_variants to use if has_params: max_variants = self.params_max_variants else: max_variants = self.path_max_variants if count >= max_variants: return False else: self._variants[clean_dict_key] = count + 1 return True
class html_comments(GrepPlugin): """ Extract and analyze HTML comments. :author: Andres Riancho ([email protected]) """ HTML_RE = re.compile('<[a-zA-Z]*.*?>.*?</[a-zA-Z]>') INTERESTING_WORDS = ( # In English 'user', 'pass', 'xxx', 'fix', 'bug', 'broken', 'oops', 'hack', 'caution', 'todo', 'note', 'warning', '!!!', '???', 'shit', 'pass', 'password', 'passwd', 'pwd', 'secret', 'stupid', # In Spanish 'tonto', 'porqueria', 'cuidado', 'usuario', u'contraseña', 'puta', 'email', 'security', 'captcha', 'pinga', 'cojones', # some in Portuguese 'banco', 'bradesco', 'itau', 'visa', 'bancoreal', u'transfêrencia', u'depósito', u'cartão', u'crédito', 'dados pessoais' ) _multi_in = multi_in([' %s ' % w for w in INTERESTING_WORDS]) def __init__(self): GrepPlugin.__init__(self) # Internal variables self._comments = DiskDict() self._already_reported_interesting = ScalableBloomFilter() def grep(self, request, response): """ Plugin entry point, parse those comments! :param request: The HTTP request object. :param response: The HTTP response object :return: None """ if not response.is_text_or_html(): return try: dp = parser_cache.dpc.get_document_parser_for(response) except BaseFrameworkException: return for comment in dp.get_comments(): # These next two lines fix this issue: # audit.ssi + grep.html_comments + web app with XSS = false positive if request.sent(comment): continue if self._is_new(comment, response): self._interesting_word(comment, request, response) self._html_in_comment(comment, request, response) def _interesting_word(self, comment, request, response): """ Find interesting words in HTML comments """ comment = comment.lower() for word in self._multi_in.query(comment): if (word, response.get_url()) not in self._already_reported_interesting: desc = 'A comment with the string "%s" was found in: "%s".'\ ' This could be interesting.' desc = desc % (word, response.get_url()) i = Info('Interesting HTML comment', desc, response.id, self.get_name()) i.set_dc(request.get_dc()) i.set_uri(response.get_uri()) i.add_to_highlight(word) kb.kb.append(self, 'interesting_comments', i) om.out.information(i.get_desc()) self._already_reported_interesting.add((word, response.get_url())) def _html_in_comment(self, comment, request, response): """ Find HTML code in HTML comments """ html_in_comment = self.HTML_RE.search(comment) if html_in_comment and \ (comment, response.get_url()) not in self._already_reported_interesting: # There is HTML code in the comment. comment = comment.strip() comment = comment.replace('\n', '') comment = comment.replace('\r', '') comment = comment[:40] desc = 'A comment with the string "%s" was found in: "%s".'\ ' This could be interesting.' desc = desc % (comment, response.get_url()) i = Info('HTML comment contains HTML code', desc, response.id, self.get_name()) i.set_dc(request.get_dc()) i.set_uri(response.get_uri()) i.add_to_highlight(html_in_comment.group(0)) kb.kb.append(self, 'html_comment_hides_html', i) om.out.information(i.get_desc()) self._already_reported_interesting.add( (comment, response.get_url())) def _is_new(self, comment, response): """ Make sure that we perform a thread safe check on the self._comments dict, in order to avoid duplicates. """ with self._plugin_lock: #pylint: disable=E1103 comment_data = self._comments.get(comment, None) if comment_data is None: self._comments[comment] = [(response.get_url(), response.id), ] return True else: if response.get_url() not in [x[0] for x in comment_data]: comment_data.append((response.get_url(), response.id)) self._comments[comment] = comment_data return True #pylint: enable=E1103 return False def end(self): """ This method is called when the plugin wont be used anymore. :return: None """ inform = [] for comment in self._comments.iterkeys(): urls_with_this_comment = self._comments[comment] stick_comment = ' '.join(comment.split()) if len(stick_comment) > 40: msg = 'A comment with the string "%s..." (and %s more bytes)'\ ' was found on these URL(s):' om.out.information( msg % (stick_comment[:40], str(len(stick_comment) - 40))) else: msg = 'A comment containing "%s" was found on these URL(s):' om.out.information(msg % (stick_comment)) for url, request_id in urls_with_this_comment: inform.append('- ' + url + ' (request with id: ' + str(request_id) + ')') inform.sort() for i in inform: om.out.information(i) self._comments.cleanup() def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class DiskDeque(object): """ The base code for this file comes from [0], I've modified it to use a DiskDict which stores the "self.data" dictionary to disk in order to save memory. [0] https://code.activestate.com/recipes/259179/ """ def __init__(self, iterable=(), maxsize=-1): if not hasattr(self, 'data'): self.left = self.right = 0 self.data = DiskDict(table_prefix='deque') self.maxsize = maxsize self.extend(iterable) def append(self, x): self.data[self.right] = x self.right += 1 if self.maxsize != -1 and len(self) > self.maxsize: self.popleft() def appendleft(self, x): self.left -= 1 self.data[self.left] = x if self.maxsize != -1 and len(self) > self.maxsize: self.pop() def pop(self): if self.left == self.right: raise IndexError('cannot pop from empty deque') self.right -= 1 elem = self.data[self.right] del self.data[self.right] return elem def popleft(self): if self.left == self.right: raise IndexError('cannot pop from empty deque') elem = self.data[self.left] del self.data[self.left] self.left += 1 return elem def clear(self): self.data.cleanup() self.left = self.right = 0 def extend(self, iterable): for elem in iterable: self.append(elem) def extendleft(self, iterable): for elem in iterable: self.appendleft(elem) def rotate(self, n=1): if self: n %= len(self) for i in xrange(n): self.appendleft(self.pop()) def __getitem__(self, i): if i < 0: i += len(self) try: return self.data[i + self.left] except KeyError: raise IndexError def __setitem__(self, i, value): if i < 0: i += len(self) try: self.data[i + self.left] = value except KeyError: raise IndexError def __delitem__(self, i): size = len(self) if not (-size <= i < size): raise IndexError data = self.data if i < 0: i += size for j in xrange(self.left+i, self.right-1): data[j] = data[j+1] self.pop() def __len__(self): return self.right - self.left def __cmp__(self, other): if type(self) != type(other): return cmp(type(self), type(other)) return cmp(list(self), list(other)) def __repr__(self, _track=[]): if id(self) in _track: return '...' _track.append(id(self)) r = 'deque(%r)' % (list(self),) _track.remove(id(self)) return r def __getstate__(self): return tuple(self) def __setstate__(self, s): self.__init__(s[0]) def __hash__(self): raise TypeError def __copy__(self): return self.__class__(self) def __deepcopy__(self, memo={}): from copy import deepcopy result = self.__class__() memo[id(self)] = result result.__init__(deepcopy(tuple(self), memo)) return result
class DiskDeque(object): """ The base code for this file comes from [0], I've modified it to use a DiskDict which stores the "self.data" dictionary to disk in order to save memory. [0] https://code.activestate.com/recipes/259179/ """ def __init__(self, iterable=(), maxsize=-1): if not hasattr(self, 'data'): self.left = self.right = 0 self.data = DiskDict(table_prefix='deque') self.maxsize = maxsize self.extend(iterable) def append(self, x): self.data[self.right] = x self.right += 1 if self.maxsize != -1 and len(self) > self.maxsize: self.popleft() def appendleft(self, x): self.left -= 1 self.data[self.left] = x if self.maxsize != -1 and len(self) > self.maxsize: self.pop() def pop(self): if self.left == self.right: raise IndexError('cannot pop from empty deque') self.right -= 1 elem = self.data[self.right] del self.data[self.right] return elem def popleft(self): if self.left == self.right: raise IndexError('cannot pop from empty deque') elem = self.data[self.left] del self.data[self.left] self.left += 1 return elem def clear(self): self.data.cleanup() self.left = self.right = 0 def extend(self, iterable): for elem in iterable: self.append(elem) def extendleft(self, iterable): for elem in iterable: self.appendleft(elem) def rotate(self, n=1): if self: n %= len(self) for i in xrange(n): self.appendleft(self.pop()) def __getitem__(self, i): if i < 0: i += len(self) try: return self.data[i + self.left] except KeyError: raise IndexError def __setitem__(self, i, value): if i < 0: i += len(self) try: self.data[i + self.left] = value except KeyError: raise IndexError def __delitem__(self, i): size = len(self) if not (-size <= i < size): raise IndexError data = self.data if i < 0: i += size for j in xrange(self.left + i, self.right - 1): data[j] = data[j + 1] self.pop() def __len__(self): return self.right - self.left def __cmp__(self, other): if type(self) != type(other): return cmp(type(self), type(other)) return cmp(list(self), list(other)) def __repr__(self, _track=[]): if id(self) in _track: return '...' _track.append(id(self)) r = 'deque(%r)' % (list(self), ) _track.remove(id(self)) return r def __getstate__(self): return tuple(self) def __setstate__(self, s): self.__init__(s[0]) def __hash__(self): raise TypeError def __copy__(self): return self.__class__(self) def __deepcopy__(self, memo={}): from copy import deepcopy result = self.__class__() memo[id(self)] = result result.__init__(deepcopy(tuple(self), memo)) return result
class CachedDiskDict(object): """ This data structure keeps the `max_in_memory` most frequently accessed keys in memory and stores the rest on disk. It is ideal for situations where a DiskDict is frequently accessed, fast read / writes are required, and items can take considerable amounts of memory. """ def __init__(self, max_in_memory=50, table_prefix=None): """ :param max_in_memory: The max number of items to keep in memory """ assert max_in_memory > 0, 'In-memory items must be > 0' table_prefix = self._get_table_prefix(table_prefix) self._max_in_memory = max_in_memory self._disk_dict = DiskDict(table_prefix=table_prefix) self._in_memory = dict() self._access_count = Counter() def cleanup(self): self._disk_dict.cleanup() def _get_table_prefix(self, table_prefix): if table_prefix is None: table_prefix = 'cached_disk_dict_%s' % rand_alpha(16) else: args = (table_prefix, rand_alpha(16)) table_prefix = 'cached_disk_dict_%s_%s' % args return table_prefix def get(self, key, default=-456): try: return self[key] except KeyError: if default is not -456: return default raise KeyError() def __getitem__(self, key): try: value = self._in_memory[key] except KeyError: # This will raise KeyError if k is not found, and that is OK # because we don't need to increase the access count when the # key doesn't exist value = self._disk_dict[key] self._increase_access_count(key) return value def _get_keys_for_memory(self): """ :return: Generate the names of the keys that should be kept in memory. For example, if `max_in_memory` is set to 2 and: _in_memory: {1: None, 2: None} _access_count: {1: 10, 2: 20, 3: 5} _disk_dict: {3: None} Then the method will generate [1, 2]. """ return [k for k, v in self._access_count.most_common(self._max_in_memory)] def _increase_access_count(self, key): self._access_count.update([key]) keys_for_memory = self._get_keys_for_memory() self._move_key_to_disk_if_needed(keys_for_memory) self._move_key_to_memory_if_needed(key, keys_for_memory) def _move_key_to_disk_if_needed(self, keys_for_memory): """ Analyzes the current access count for the last accessed key and checks if any if the keys in memory should be moved to disk. :param keys_for_memory: The keys that should be in memory :return: The name of the key that was moved to disk, or None if all the keys are still in memory. """ for key in self._in_memory: if key in keys_for_memory: continue try: value = self._in_memory.pop(key) except KeyError: return else: self._disk_dict[key] = value return key def _move_key_to_memory_if_needed(self, key, keys_for_memory): """ Analyzes the current access count for the last accessed key and checks if any if the keys in disk should be moved to memory. :param key: The key that was last accessed :param keys_for_memory: The keys that should be in memory :return: The name of the key that was moved to memory, or None if all the keys are still on disk. """ # The key is already in memory, nothing to do here if key in self._in_memory: return # The key must not be in memory, nothing to do here if key not in keys_for_memory: return try: value = self._disk_dict.pop(key) except KeyError: return else: self._in_memory[key] = value return key def __setitem__(self, key, value): if key in self._in_memory: self._in_memory[key] = value elif len(self._in_memory) < self._max_in_memory: self._in_memory[key] = value else: self._disk_dict[key] = value self._increase_access_count(key) def __delitem__(self, key): try: del self._in_memory[key] except KeyError: # This will raise KeyError if k is not found, and that is OK # because we don't need to increase the access count when the # key doesn't exist del self._disk_dict[key] try: del self._access_count[key] except KeyError: # Another thread removed this key pass def __contains__(self, key): if key in self._in_memory: self._increase_access_count(key) return True if key in self._disk_dict: self._increase_access_count(key) return True return False def __iter__(self): """ Decided not to increase the access count when iterating through the items. In most cases the iteration will be performed on all items, thus increasing the access count +1 for each, which will leave all access counts +1, forcing no movements between memory and disk. """ for key in self._in_memory: yield key for key in self._disk_dict: yield key def iteritems(self): for key, value in self._in_memory.iteritems(): yield key, value for key, value in self._disk_dict.iteritems(): yield key, value
class ssi(AuditPlugin): """ Find server side inclusion vulnerabilities. :author: Andres Riancho ([email protected]) """ def __init__(self): AuditPlugin.__init__(self) # Internal variables self._expected_res_mutant = DiskDict() self._freq_list = DiskList() re_str = '<!--#exec cmd="echo -n (.*?);echo -n (.*?)" -->' self._extract_results_re = re.compile(re_str) def audit(self, freq, orig_response): """ Tests an URL for server side inclusion vulnerabilities. :param freq: A FuzzableRequest """ # Create the mutants to send right now, ssi_strings = self._get_ssi_strings() mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response) # Used in end() to detect "persistent SSI" for mut in mutants: expected_result = self._extract_result_from_payload( mut.get_mod_value()) self._expected_res_mutant[expected_result] = mut self._freq_list.append(freq) # End of persistent SSI setup self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants, self._analyze_result) def _get_ssi_strings(self): """ This method returns a list of server sides to try to include. :return: A string, see above. """ yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % (rand_alpha(5), rand_alpha(5)) # TODO: Add mod_perl ssi injection support # http://www.sens.buffalo.edu/services/webhosting/advanced/perlssi.shtml #yield <!--#perl sub="sub {print qq/If you see this, mod_perl is working!/;}" --> def _extract_result_from_payload(self, payload): """ Extract the expected result from the payload we're sending. """ match = self._extract_results_re.search(payload) return match.group(1) + match.group(2) def _analyze_result(self, mutant, response): """ Analyze the result of the previously sent request. :return: None, save the vuln to the kb. """ if self._has_no_bug(mutant): e_res = self._extract_result_from_payload(mutant.get_mod_value()) if e_res in response and not e_res in mutant.get_original_response_body( ): desc = 'Server side include (SSI) was found at: %s' desc = desc % mutant.found_at() v = Vuln.from_mutant('Server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(e_res) self.kb_append_uniq(self, 'ssi', v) def end(self): """ This method is called when the plugin wont be used anymore and is used to find persistent SSI vulnerabilities. Example where a persistent SSI can be found: Say you have a "guestbook" (a CGI application that allows visitors to leave messages for everyone to see) on a server that has SSI enabled. Most such guestbooks around the Net actually allow visitors to enter HTML code as part of their comments. Now, what happens if a malicious visitor decides to do some damage by entering the following: <!--#exec cmd="ls" --> If the guestbook CGI program was designed carefully, to strip SSI commands from the input, then there is no problem. But, if it was not, there exists the potential for a major headache! For a working example please see moth VM. """ multi_in_inst = multi_in(self._expected_res_mutant.keys()) def filtered_freq_generator(freq_list): already_tested = ScalableBloomFilter() for freq in freq_list: if freq not in already_tested: already_tested.add(freq) yield freq def analyze_persistent(freq, response): for matched_expected_result in multi_in_inst.query( response.get_body()): # We found one of the expected results, now we search the # self._persistent_data to find which of the mutants sent it # and create the vulnerability mutant = self._expected_res_mutant[matched_expected_result] desc = 'Server side include (SSI) was found at: %s' \ ' The result of that injection is shown by browsing'\ ' to "%s".' desc = desc % (mutant.found_at(), freq.get_url()) v = Vuln.from_mutant( 'Persistent server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(matched_expected_result) self.kb_append(self, 'ssi', v) self._send_mutants_in_threads(self._uri_opener.send_mutant, filtered_freq_generator(self._freq_list), analyze_persistent, cache=False) self._expected_res_mutant.cleanup() self._freq_list.cleanup() def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class ssi(AuditPlugin): """ Find server side inclusion vulnerabilities. :author: Andres Riancho ([email protected]) """ def __init__(self): AuditPlugin.__init__(self) # Internal variables self._expected_mutant_dict = DiskDict(table_prefix='ssi') self._extract_expected_re = re.compile('[1-9]{5}') def audit(self, freq, orig_response): """ Tests an URL for server side inclusion vulnerabilities. :param freq: A FuzzableRequest """ ssi_strings = [] for string in self._get_ssi_strings(): ssi_strings.append(string) mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response) self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants, self._analyze_result) def _get_ssi_strings(self): """ This method returns a list of server sides to try to include. :return: A string, see above. """ # Generic # yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % get_seeds() # Perl SSI # yield ('<!--#set var="SEED_A" value="%s" -->' # '<!--#echo var="SEED_A" -->' # '<!--#set var="SEED_B" value="%s" -->' # '<!--#echo var="SEED_B" -->' % get_seeds()) # Smarty # http://www.smarty.net/docsv2/en/language.function.math.tpl yield '{math equation="x * y" x=%s y=%s}' % get_seeds() # Mako # http://docs.makotemplates.org/en/latest/syntax.html yield '${%s * %s}' % get_seeds() # Jinja2 and Twig # http://jinja.pocoo.org/docs/dev/templates/#math # http://twig.sensiolabs.org/doc/templates.html yield '{{%s * %s}}' % get_seeds() # Generic yield '{%s * %s}' % get_seeds() # OGNL (struts2) #yield '${%%{%s * %s}}' % get_seeds() def _get_expected_results(self, mutant): """ Extracts the potential results from the mutant payload and returns them in a list. """ sent_payload = mutant.get_token_payload() seed_numbers = self._extract_expected_re.findall(sent_payload) seed_a = int(seed_numbers[0]) seed_b = int(seed_numbers[1]) return [str(seed_a * seed_b), '%s%s' % (seed_a, seed_b)] def _analyze_result(self, mutant, response): """ Analyze the result of the previously sent request. :return: None, save the vuln to the kb. """ # Store the mutants in order to be able to analyze the persistent case # later expected_results = self._get_expected_results(mutant) for expected_result in expected_results: self._expected_mutant_dict[expected_result] = mutant # Now we analyze the "reflected" case if self._has_bug(mutant): return for expected_result in expected_results: if expected_result not in response: continue if expected_result in mutant.get_original_response_body(): continue desc = 'Server side include (SSI) was found at: %s' desc %= mutant.found_at() v = Vuln.from_mutant('Server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(expected_result) self.kb_append_uniq(self, 'ssi', v) def end(self): """ This method is called when the plugin wont be used anymore and is used to find persistent SSI vulnerabilities. Example where a persistent SSI can be found: Say you have a "guest book" (a CGI application that allows visitors to leave messages for everyone to see) on a server that has SSI enabled. Most such guest books around the Net actually allow visitors to enter HTML code as part of their comments. Now, what happens if a malicious visitor decides to do some damage by entering the following: <!--#exec cmd="ls" --> If the guest book CGI program was designed carefully, to strip SSI commands from the input, then there is no problem. But, if it was not, there exists the potential for a major headache! For a working example please see moth VM. """ fuzzable_request_set = kb.kb.get_all_known_fuzzable_requests() self._send_mutants_in_threads(self._uri_opener.send_mutant, fuzzable_request_set, self._analyze_persistent, cache=False) self._expected_mutant_dict.cleanup() def _analyze_persistent(self, freq, response): """ Analyze the response of sending each fuzzable request found by the framework, trying to identify any locations where we might have injected a payload. :param freq: The fuzzable request :param response: The HTTP response :return: None, vulns are stored in KB """ multi_in_inst = multi_in(self._expected_mutant_dict.keys()) for matched_expected_result in multi_in_inst.query( response.get_body()): # We found one of the expected results, now we search the # self._expected_mutant_dict to find which of the mutants sent it # and create the vulnerability mutant = self._expected_mutant_dict[matched_expected_result] desc = ('Server side include (SSI) was found at: %s' ' The result of that injection is shown by browsing' ' to "%s".') desc %= (mutant.found_at(), freq.get_url()) v = Vuln.from_mutant( 'Persistent server side include vulnerability', desc, severity.HIGH, response.id, self.get_name(), mutant) v.add_to_highlight(matched_expected_result) self.kb_append(self, 'ssi', v) def get_long_desc(self): """ :return: A DETAILED description of the plugin functions and features. """ return """
class CachedDiskDict(object): """ This data structure keeps the `max_in_memory` most frequently accessed keys in memory and stores the rest on disk. It is ideal for situations where a DiskDict is frequently accessed, fast read / writes are required, and items can take considerable amounts of memory. """ def __init__(self, max_in_memory=50, table_prefix=None): """ :param max_in_memory: The max number of items to keep in memory """ assert max_in_memory > 0, 'In-memory items must be > 0' table_prefix = self._get_table_prefix(table_prefix) self._max_in_memory = max_in_memory self._disk_dict = DiskDict(table_prefix=table_prefix) self._in_memory = dict() self._access_count = dict() def cleanup(self): self._disk_dict.cleanup() def _get_table_prefix(self, table_prefix): if table_prefix is None: table_prefix = 'cached_disk_dict_%s' % rand_alpha(16) else: args = (table_prefix, rand_alpha(16)) table_prefix = 'cached_disk_dict_%s_%s' % args return table_prefix def get(self, key, default=-456): try: return self[key] except KeyError: if default is not -456: return default raise KeyError() def __getitem__(self, key): try: value = self._in_memory[key] except KeyError: # This will raise KeyError if k is not found, and that is OK # because we don't need to increase the access count when the # key doesn't exist value = self._disk_dict[key] self._increase_access_count(key) return value def _get_keys_for_memory(self): """ :return: Generate the names of the keys that should be kept in memory. For example, if `max_in_memory` is set to 2 and: _in_memory: {1: None, 2: None} _access_count: {1: 10, 2: 20, 3: 5} _disk_dict: {3: None} Then the method will generate [1, 2]. """ items = self._access_count.items() items.sort(sort_by_value) iterator = min(self._max_in_memory, len(items)) for i in xrange(iterator): yield items[i][0] def _belongs_in_memory(self, key): """ :param key: A key :return: True if the key should be stored in memory """ if key in self._get_keys_for_memory(): return True return False def _increase_access_count(self, key): access_count = self._access_count.get(key, 0) access_count += 1 self._access_count[key] = access_count self._move_key_to_disk_if_needed(key) self._move_key_to_memory_if_needed(key) def _move_key_to_disk_if_needed(self, key): """ Analyzes the current access count for the last accessed key and checks if any if the keys in memory should be moved to disk. :param key: The key that was last accessed :return: The name of the key that was moved to disk, or None if all the keys are still in memory. """ for key in self._in_memory.keys(): if not self._belongs_in_memory(key): try: value = self._in_memory[key] except KeyError: return None else: self._disk_dict[key] = value self._in_memory.pop(key, None) return key def _move_key_to_memory_if_needed(self, key): """ Analyzes the current access count for the last accessed key and checks if any if the keys in disk should be moved to memory. :param key: The key that was last accessed :return: The name of the key that was moved to memory, or None if all the keys are still on disk. """ key_belongs_in_memory = self._belongs_in_memory(key) if not key_belongs_in_memory: return None try: value = self._disk_dict[key] except KeyError: return None else: self._in_memory[key] = value self._disk_dict.pop(key, None) return key def __setitem__(self, key, value): if len(self._in_memory) < self._max_in_memory: self._in_memory[key] = value else: self._disk_dict[key] = value self._increase_access_count(key) def __delitem__(self, key): try: del self._in_memory[key] except KeyError: # This will raise KeyError if k is not found, and that is OK # because we don't need to increase the access count when the # key doesn't exist del self._disk_dict[key] try: del self._access_count[key] except KeyError: # Another thread removed this key pass def __contains__(self, key): if key in self._in_memory: self._increase_access_count(key) return True if key in self._disk_dict: self._increase_access_count(key) return True return False def __iter__(self): """ Decided not to increase the access count when iterating through the items. In most cases the iteration will be performed on all items, thus increasing the access count +1 for each, which will leave all access counts +1, forcing no movements between memory and disk. """ for key in self._in_memory: yield key for key in self._disk_dict: yield key