def test_remove_table(self):
disk_dict = DiskDict()
table_name = disk_dict.table_name
db = get_default_temp_db_instance()
self.assertTrue(db.table_exists(table_name))
disk_dict.cleanup()
self.assertFalse(db.table_exists(table_name))
def test_remove_table(self):
disk_dict = DiskDict()
table_name = disk_dict.table_name
db = get_default_temp_db_instance()
self.assertTrue(db.table_exists(table_name))
disk_dict.cleanup()
self.assertFalse(db.table_exists(table_name))
def test_table_with_prefix(self):
_unittest = 'unittest'
disk_dict = DiskDict(_unittest)
self.assertIn(_unittest, disk_dict.table_name)
db = get_default_temp_db_instance()
self.assertTrue(db.table_exists(disk_dict.table_name))
disk_dict.cleanup()
self.assertFalse(db.table_exists(disk_dict.table_name))
def test_table_with_prefix(self):
_unittest = 'unittest'
disk_dict = DiskDict(_unittest)
self.assertIn(_unittest, disk_dict.table_name)
db = get_default_temp_db_instance()
self.assertTrue(db.table_exists(disk_dict.table_name))
disk_dict.cleanup()
self.assertFalse(db.table_exists(disk_dict.table_name))
class html_comments(GrepPlugin):
"""
Extract and analyze HTML comments.
:author: Andres Riancho ([email protected])
"""
HTML_RE = re.compile('<[a-zA-Z]*.*?>.*?</[a-zA-Z]>')
INTERESTING_WORDS = (
# In English
'user',
'pass',
'xxx',
'fix',
'bug',
'broken',
'oops',
'hack',
'caution',
'todo',
'note',
'warning',
'!!!',
'???',
'shit',
'pass',
'password',
'passwd',
'pwd',
'secret',
'stupid',
# In Spanish
'tonto',
'porqueria',
'cuidado',
'usuario',
u'contraseña',
'puta',
'email',
'security',
'captcha',
'pinga',
'cojones',
# some in Portuguese
'banco',
'bradesco',
'itau',
'visa',
'bancoreal',
u'transfêrencia',
u'depósito',
u'cartão',
u'crédito',
'dados pessoais')
_multi_in = multi_in([' %s ' % w for w in INTERESTING_WORDS])
def __init__(self):
GrepPlugin.__init__(self)
# Internal variables
self._comments = DiskDict(table_prefix='html_comments')
self._already_reported = ScalableBloomFilter()
def grep(self, request, response):
"""
Plugin entry point, parse those comments!
:param request: The HTTP request object.
:param response: The HTTP response object
:return: None
"""
if not response.is_text_or_html():
return
try:
dp = parser_cache.dpc.get_document_parser_for(response)
except BaseFrameworkException:
return
for comment in dp.get_comments():
# These next two lines fix this issue:
# audit.ssi + grep.html_comments + web app with XSS = false positive
if request.sent(comment):
continue
if self._is_new(comment, response):
self._interesting_word(comment, request, response)
self._html_in_comment(comment, request, response)
def _interesting_word(self, comment, request, response):
"""
Find interesting words in HTML comments
"""
comment = comment.lower()
for word in self._multi_in.query(comment):
if (word, response.get_url()) in self._already_reported:
continue
desc = ('A comment with the string "%s" was found in: "%s".'
' This could be interesting.')
desc %= (word, response.get_url())
v = Vuln.from_fr('Interesting HTML comment',
desc, severity.INFORMATION, response.id,
self.get_name(), request)
v.add_to_highlight(word)
kb.kb.append(self, 'interesting_comments', v)
self._already_reported.add((word, response.get_url()))
def _html_in_comment(self, comment, request, response):
"""
Find HTML code in HTML comments
"""
html_in_comment = self.HTML_RE.search(comment)
if html_in_comment is None:
return
if (comment, response.get_url()) in self._already_reported:
return
# There is HTML code in the comment.
comment = comment.strip()
comment = comment.replace('\n', '')
comment = comment.replace('\r', '')
comment = comment[:40]
desc = ('A comment with the string "%s" was found in: "%s".'
' This could be interesting.')
desc %= (comment, response.get_url())
v = Vuln.from_fr('HTML comment contains HTML code',
desc, severity.INFORMATION, response.id,
self.get_name(), request)
v.set_uri(response.get_uri())
v.add_to_highlight(html_in_comment.group(0))
om.out.vulnerability(v.get_desc(), severity=severity.INFORMATION)
kb.kb.append(self, 'html_comment_hides_html', v)
self._already_reported.add((comment, response.get_url()))
def _is_new(self, comment, response):
"""
Make sure that we perform a thread safe check on the self._comments
dict, in order to avoid duplicates.
"""
with self._plugin_lock:
#pylint: disable=E1103
comment_data = self._comments.get(comment, None)
response_url = response.get_url()
if comment_data is None:
self._comments[comment] = [(response_url, response.id)]
return True
else:
for saved_url, response_id in comment_data:
if response_url == saved_url:
return False
else:
comment_data.append((response_url, response.id))
self._comments[comment] = comment_data
return True
#pylint: enable=E1103
def end(self):
"""
This method is called when the plugin wont be used anymore.
:return: None
"""
for comment, url_request_id_lst in self._comments.iteritems():
stick_comment = ' '.join(comment.split())
if len(stick_comment) > 40:
msg = ('A comment with the string "%s..." (and %s more bytes)'
' was found on these URL(s):')
args = (stick_comment[:40], str(len(stick_comment) - 40))
om.out.vulnerability(msg % args, severity=severity.INFORMATION)
else:
msg = 'A comment containing "%s" was found on these URL(s):'
om.out.vulnerability(msg % stick_comment,
severity=severity.INFORMATION)
inform = []
for url, request_id in url_request_id_lst:
msg = '- %s (request with id: %s)'
inform.append(msg % (url, request_id))
for i in sorted(inform):
om.out.vulnerability(i, severity=severity.INFORMATION)
self._comments.cleanup()
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class VariantDB(object):
"""
See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also
understand that we'll keep "dirty" versions of the references/fuzzable
requests in order to be able to answer "False" to a call for
need_more_variants in a situation like this:
>> need_more_variants('http://foo.com/abc?id=32')
True
>> append('http://foo.com/abc?id=32')
True
>> need_more_variants('http://foo.com/abc?id=32')
False
"""
HASH_IGNORE_HEADERS = ('referer', )
TAG = '[variant_db]'
def __init__(self):
self._variants = DiskDict(table_prefix='variant_db')
self._variants_eq = ScalableBloomFilter()
self._variants_form = DiskDict(table_prefix='variant_db_form')
self.params_max_variants = cf.cf.get('params_max_variants')
self.path_max_variants = cf.cf.get('path_max_variants')
self.max_equal_form_variants = cf.cf.get('max_equal_form_variants')
self._db_lock = threading.RLock()
def cleanup(self):
self._variants.cleanup()
self._variants_form.cleanup()
def append(self, fuzzable_request):
"""
:return: True if we added a new fuzzable request variant to the DB,
False if NO more variants are required for this fuzzable
request.
"""
with self._db_lock:
if self._seen_exactly_the_same(fuzzable_request):
return False
if self._has_form(fuzzable_request):
if not self._need_more_variants_for_form(fuzzable_request):
return False
if not self._need_more_variants_for_uri(fuzzable_request):
return False
# Yes, please give me more variants of fuzzable_request
return True
def _log_return_false(self, fuzzable_request, reason):
args = (reason, fuzzable_request)
msg = 'VariantDB is returning False because of "%s" for "%s"'
om.out.debug(msg % args)
def _need_more_variants_for_uri(self, fuzzable_request):
#
# Do we need more variants for the fuzzable request? (similar match)
# PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS
#
clean_dict_key = clean_fuzzable_request(fuzzable_request)
count = self._variants.get(clean_dict_key, None)
if count is None:
self._variants[clean_dict_key] = 1
return True
# We've seen at least one fuzzable request with this pattern...
url = fuzzable_request.get_uri()
has_params = url.has_query_string() or fuzzable_request.get_raw_data()
# Choose which max_variants to use
if has_params:
max_variants = self.params_max_variants
max_variants_type = 'params'
else:
max_variants = self.path_max_variants
max_variants_type = 'path'
if count >= max_variants:
_type = 'need_more_variants_for_uri(%s)' % max_variants_type
self._log_return_false(fuzzable_request, _type)
return False
self._variants[clean_dict_key] = count + 1
return True
def _seen_exactly_the_same(self, fuzzable_request):
#
# Is the fuzzable request already known to us? (exactly the same)
#
request_hash = fuzzable_request.get_request_hash(
self.HASH_IGNORE_HEADERS)
if request_hash in self._variants_eq:
return True
# Store it to avoid duplicated fuzzable requests in our framework
self._variants_eq.add(request_hash)
self._log_return_false(fuzzable_request, 'seen_exactly_the_same')
return False
def _has_form(self, fuzzable_request):
raw_data = fuzzable_request.get_raw_data()
if raw_data and len(raw_data.get_param_names()) >= 2:
return True
return False
def _need_more_variants_for_form(self, fuzzable_request):
#
# Do we need more variants for this form? (similar match)
# MAX_EQUAL_FORM_VARIANTS
#
clean_dict_key_form = clean_fuzzable_request_form(fuzzable_request)
count = self._variants_form.get(clean_dict_key_form, None)
if count is None:
self._variants_form[clean_dict_key_form] = 1
return True
if count >= self.max_equal_form_variants:
self._log_return_false(fuzzable_request,
'need_more_variants_for_form')
return False
self._variants_form[clean_dict_key_form] = count + 1
return True
class VariantDB(object):
"""
See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also
understand that we'll keep "dirty" versions of the references/fuzzable
requests in order to be able to answer "False" to a call for
need_more_variants in a situation like this:
need_more_variants('http://foo.com/abc?id=32') --> True
append('http://foo.com/abc?id=32')
need_more_variants('http://foo.com/abc?id=32') --> False
"""
HASH_IGNORE_HEADERS = ('referer', )
TAG = '[variant_db]'
def __init__(self,
params_max_variants=PARAMS_MAX_VARIANTS,
path_max_variants=PATH_MAX_VARIANTS):
self._variants_eq = DiskDict(table_prefix='variant_db_eq')
self._variants = DiskDict(table_prefix='variant_db')
self.params_max_variants = params_max_variants
self.path_max_variants = path_max_variants
self._db_lock = threading.RLock()
def cleanup(self):
self._variants_eq.cleanup()
self._variants.cleanup()
def append(self, fuzzable_request):
"""
:return: True if we added a new fuzzable request variant to the DB,
False if no more variants are required for this fuzzable
request.
"""
with self._db_lock:
#
# Is the fuzzable request already known to us? (exactly the same)
#
request_hash = fuzzable_request.get_request_hash(
self.HASH_IGNORE_HEADERS)
already_seen = self._variants_eq.get(request_hash, False)
if already_seen:
return False
# Store it to avoid duplicated fuzzable requests in our framework
self._variants_eq[request_hash] = True
#
# Do we need more variants for the fuzzable request? (similar match)
#
clean_dict_key = clean_fuzzable_request(fuzzable_request)
count = self._variants.get(clean_dict_key, None)
if count is None:
self._variants[clean_dict_key] = 1
return True
# We've seen at least one fuzzable request with this pattern...
url = fuzzable_request.get_uri()
has_params = url.has_query_string(
) or fuzzable_request.get_raw_data()
# Choose which max_variants to use
if has_params:
max_variants = self.params_max_variants
else:
max_variants = self.path_max_variants
if count >= max_variants:
return False
else:
self._variants[clean_dict_key] = count + 1
return True
class ssi(AuditPlugin):
"""
Find server side inclusion vulnerabilities.
:author: Andres Riancho ([email protected])
"""
def __init__(self):
AuditPlugin.__init__(self)
# Internal variables
self._expected_res_mutant = DiskDict()
self._freq_list = DiskList()
re_str = '<!--#exec cmd="echo -n (.*?);echo -n (.*?)" -->'
self._extract_results_re = re.compile(re_str)
def audit(self, freq, orig_response):
"""
Tests an URL for server side inclusion vulnerabilities.
:param freq: A FuzzableRequest
"""
# Create the mutants to send right now,
ssi_strings = self._get_ssi_strings()
mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response)
# Used in end() to detect "persistent SSI"
for mut in mutants:
expected_result = self._extract_result_from_payload(
mut.get_token_value())
self._expected_res_mutant[expected_result] = mut
self._freq_list.append(freq)
# End of persistent SSI setup
self._send_mutants_in_threads(self._uri_opener.send_mutant,
mutants,
self._analyze_result)
def _get_ssi_strings(self):
"""
This method returns a list of server sides to try to include.
:return: A string, see above.
"""
yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % (rand_alpha(5),
rand_alpha(5))
# TODO: Add mod_perl ssi injection support
# http://www.sens.buffalo.edu/services/webhosting/advanced/perlssi.shtml
#yield <!--#perl sub="sub {print qq/If you see this, mod_perl is working!/;}" -->
def _extract_result_from_payload(self, payload):
"""
Extract the expected result from the payload we're sending.
"""
match = self._extract_results_re.search(payload)
return match.group(1) + match.group(2)
def _analyze_result(self, mutant, response):
"""
Analyze the result of the previously sent request.
:return: None, save the vuln to the kb.
"""
if self._has_no_bug(mutant):
e_res = self._extract_result_from_payload(mutant.get_token_value())
if e_res in response and not e_res in mutant.get_original_response_body():
desc = 'Server side include (SSI) was found at: %s'
desc = desc % mutant.found_at()
v = Vuln.from_mutant('Server side include vulnerability', desc,
severity.HIGH, response.id, self.get_name(),
mutant)
v.add_to_highlight(e_res)
self.kb_append_uniq(self, 'ssi', v)
def end(self):
"""
This method is called when the plugin wont be used anymore and is used
to find persistent SSI vulnerabilities.
Example where a persistent SSI can be found:
Say you have a "guestbook" (a CGI application that allows visitors
to leave messages for everyone to see) on a server that has SSI
enabled. Most such guestbooks around the Net actually allow visitors
to enter HTML code as part of their comments. Now, what happens if a
malicious visitor decides to do some damage by entering the following:
<!--#exec cmd="ls" -->
If the guestbook CGI program was designed carefully, to strip SSI
commands from the input, then there is no problem. But, if it was not,
there exists the potential for a major headache!
For a working example please see moth VM.
"""
multi_in_inst = multi_in(self._expected_res_mutant.keys())
def filtered_freq_generator(freq_list):
already_tested = ScalableBloomFilter()
for freq in freq_list:
if freq not in already_tested:
already_tested.add(freq)
yield freq
def analyze_persistent(freq, response):
for matched_expected_result in multi_in_inst.query(response.get_body()):
# We found one of the expected results, now we search the
# self._persistent_data to find which of the mutants sent it
# and create the vulnerability
mutant = self._expected_res_mutant[matched_expected_result]
desc = 'Server side include (SSI) was found at: %s' \
' The result of that injection is shown by browsing'\
' to "%s".'
desc = desc % (mutant.found_at(), freq.get_url())
v = Vuln.from_mutant('Persistent server side include vulnerability',
desc, severity.HIGH, response.id,
self.get_name(), mutant)
v.add_to_highlight(matched_expected_result)
self.kb_append(self, 'ssi', v)
self._send_mutants_in_threads(self._uri_opener.send_mutant,
filtered_freq_generator(self._freq_list),
analyze_persistent,
cache=False)
self._expected_res_mutant.cleanup()
self._freq_list.cleanup()
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class ssi(AuditPlugin):
"""
Find server side inclusion vulnerabilities.
:author: Andres Riancho ([email protected])
"""
def __init__(self):
AuditPlugin.__init__(self)
# Internal variables
self._expected_mutant_dict = DiskDict(table_prefix="ssi")
self._extract_expected_re = re.compile("[1-9]{5}")
def audit(self, freq, orig_response):
"""
Tests an URL for server side inclusion vulnerabilities.
:param freq: A FuzzableRequest
"""
ssi_strings = self._get_ssi_strings()
mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response)
self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants, self._analyze_result)
def _get_ssi_strings(self):
"""
This method returns a list of server sides to try to include.
:return: A string, see above.
"""
# Generic
yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % get_seeds()
# Perl SSI
yield (
'<!--#set var="SEED_A" value="%s" -->'
'<!--#echo var="SEED_A" -->'
'<!--#set var="SEED_B" value="%s" -->'
'<!--#echo var="SEED_B" -->' % get_seeds()
)
# Smarty
# http://www.smarty.net/docsv2/en/language.function.math.tpl
yield '{math equation="x * y" x=%s y=%s}' % get_seeds()
# Mako
# http://docs.makotemplates.org/en/latest/syntax.html
yield "${%s * %s}" % get_seeds()
# Jinja2 and Twig
# http://jinja.pocoo.org/docs/dev/templates/#math
# http://twig.sensiolabs.org/doc/templates.html
yield "{{%s * %s}}" % get_seeds()
# Generic
yield "{%s * %s}" % get_seeds()
def _get_expected_results(self, mutant):
"""
Extracts the potential results from the mutant payload and returns them
in a list.
"""
sent_payload = mutant.get_token_payload()
seed_numbers = self._extract_expected_re.findall(sent_payload)
seed_a = int(seed_numbers[0])
seed_b = int(seed_numbers[1])
return [str(seed_a * seed_b), "%s%s" % (seed_a, seed_b)]
def _analyze_result(self, mutant, response):
"""
Analyze the result of the previously sent request.
:return: None, save the vuln to the kb.
"""
# Store the mutants in order to be able to analyze the persistent case
# later
expected_results = self._get_expected_results(mutant)
for expected_result in expected_results:
self._expected_mutant_dict[expected_result] = mutant
# Now we analyze the "reflected" case
if self._has_bug(mutant):
return
for expected_result in expected_results:
if expected_result not in response:
continue
if expected_result in mutant.get_original_response_body():
continue
desc = "Server side include (SSI) was found at: %s"
desc %= mutant.found_at()
v = Vuln.from_mutant(
"Server side include vulnerability", desc, severity.HIGH, response.id, self.get_name(), mutant
)
v.add_to_highlight(expected_result)
self.kb_append_uniq(self, "ssi", v)
def end(self):
"""
This method is called when the plugin wont be used anymore and is used
to find persistent SSI vulnerabilities.
Example where a persistent SSI can be found:
Say you have a "guest book" (a CGI application that allows visitors
to leave messages for everyone to see) on a server that has SSI
enabled. Most such guest books around the Net actually allow visitors
to enter HTML code as part of their comments. Now, what happens if a
malicious visitor decides to do some damage by entering the following:
<!--#exec cmd="ls" -->
If the guest book CGI program was designed carefully, to strip SSI
commands from the input, then there is no problem. But, if it was not,
there exists the potential for a major headache!
For a working example please see moth VM.
"""
fuzzable_request_set = kb.kb.get_all_known_fuzzable_requests()
self._send_mutants_in_threads(
self._uri_opener.send_mutant, fuzzable_request_set, self._analyze_persistent, cache=False
)
self._expected_mutant_dict.cleanup()
def _analyze_persistent(self, freq, response):
"""
Analyze the response of sending each fuzzable request found by the
framework, trying to identify any locations where we might have injected
a payload.
:param freq: The fuzzable request
:param response: The HTTP response
:return: None, vulns are stored in KB
"""
multi_in_inst = multi_in(self._expected_mutant_dict.keys())
for matched_expected_result in multi_in_inst.query(response.get_body()):
# We found one of the expected results, now we search the
# self._expected_mutant_dict to find which of the mutants sent it
# and create the vulnerability
mutant = self._expected_mutant_dict[matched_expected_result]
desc = (
"Server side include (SSI) was found at: %s"
" The result of that injection is shown by browsing"
' to "%s".'
)
desc %= (mutant.found_at(), freq.get_url())
v = Vuln.from_mutant(
"Persistent server side include vulnerability",
desc,
severity.HIGH,
response.id,
self.get_name(),
mutant,
)
v.add_to_highlight(matched_expected_result)
self.kb_append(self, "ssi", v)
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class VariantDB(object):
"""
See the notes on PARAMS_MAX_VARIANTS and PATH_MAX_VARIANTS above. Also
understand that we'll keep "dirty" versions of the references/fuzzable
requests in order to be able to answer "False" to a call for
need_more_variants in a situation like this:
need_more_variants('http://foo.com/abc?id=32') --> True
append('http://foo.com/abc?id=32')
need_more_variants('http://foo.com/abc?id=32') --> False
"""
HASH_IGNORE_HEADERS = ('referer',)
TAG = '[variant_db]'
def __init__(self, params_max_variants=PARAMS_MAX_VARIANTS,
path_max_variants=PATH_MAX_VARIANTS):
self._variants_eq = DiskDict(table_prefix='variant_db_eq')
self._variants = DiskDict(table_prefix='variant_db')
self.params_max_variants = params_max_variants
self.path_max_variants = path_max_variants
self._db_lock = threading.RLock()
def cleanup(self):
self._variants_eq.cleanup()
self._variants.cleanup()
def append(self, fuzzable_request):
"""
:return: True if we added a new fuzzable request variant to the DB,
False if no more variants are required for this fuzzable
request.
"""
with self._db_lock:
#
# Is the fuzzable request already known to us? (exactly the same)
#
request_hash = fuzzable_request.get_request_hash(self.HASH_IGNORE_HEADERS)
already_seen = self._variants_eq.get(request_hash, False)
if already_seen:
return False
# Store it to avoid duplicated fuzzable requests in our framework
self._variants_eq[request_hash] = True
#
# Do we need more variants for the fuzzable request? (similar match)
#
clean_dict_key = clean_fuzzable_request(fuzzable_request)
count = self._variants.get(clean_dict_key, None)
if count is None:
self._variants[clean_dict_key] = 1
return True
# We've seen at least one fuzzable request with this pattern...
url = fuzzable_request.get_uri()
has_params = url.has_query_string() or fuzzable_request.get_raw_data()
# Choose which max_variants to use
if has_params:
max_variants = self.params_max_variants
else:
max_variants = self.path_max_variants
if count >= max_variants:
return False
else:
self._variants[clean_dict_key] = count + 1
return True
class html_comments(GrepPlugin):
"""
Extract and analyze HTML comments.
:author: Andres Riancho ([email protected])
"""
HTML_RE = re.compile('<[a-zA-Z]*.*?>.*?</[a-zA-Z]>')
INTERESTING_WORDS = (
# In English
'user', 'pass', 'xxx', 'fix', 'bug', 'broken', 'oops', 'hack',
'caution', 'todo', 'note', 'warning', '!!!', '???', 'shit',
'pass', 'password', 'passwd', 'pwd', 'secret', 'stupid',
# In Spanish
'tonto', 'porqueria', 'cuidado', 'usuario', u'contraseña',
'puta', 'email', 'security', 'captcha', 'pinga', 'cojones',
# some in Portuguese
'banco', 'bradesco', 'itau', 'visa', 'bancoreal', u'transfêrencia',
u'depósito', u'cartão', u'crédito', 'dados pessoais'
)
_multi_in = multi_in([' %s ' % w for w in INTERESTING_WORDS])
def __init__(self):
GrepPlugin.__init__(self)
# Internal variables
self._comments = DiskDict()
self._already_reported_interesting = ScalableBloomFilter()
def grep(self, request, response):
"""
Plugin entry point, parse those comments!
:param request: The HTTP request object.
:param response: The HTTP response object
:return: None
"""
if not response.is_text_or_html():
return
try:
dp = parser_cache.dpc.get_document_parser_for(response)
except BaseFrameworkException:
return
for comment in dp.get_comments():
# These next two lines fix this issue:
# audit.ssi + grep.html_comments + web app with XSS = false positive
if request.sent(comment):
continue
if self._is_new(comment, response):
self._interesting_word(comment, request, response)
self._html_in_comment(comment, request, response)
def _interesting_word(self, comment, request, response):
"""
Find interesting words in HTML comments
"""
comment = comment.lower()
for word in self._multi_in.query(comment):
if (word, response.get_url()) not in self._already_reported_interesting:
desc = 'A comment with the string "%s" was found in: "%s".'\
' This could be interesting.'
desc = desc % (word, response.get_url())
i = Info('Interesting HTML comment', desc,
response.id, self.get_name())
i.set_dc(request.get_dc())
i.set_uri(response.get_uri())
i.add_to_highlight(word)
kb.kb.append(self, 'interesting_comments', i)
om.out.information(i.get_desc())
self._already_reported_interesting.add((word,
response.get_url()))
def _html_in_comment(self, comment, request, response):
"""
Find HTML code in HTML comments
"""
html_in_comment = self.HTML_RE.search(comment)
if html_in_comment and \
(comment, response.get_url()) not in self._already_reported_interesting:
# There is HTML code in the comment.
comment = comment.strip()
comment = comment.replace('\n', '')
comment = comment.replace('\r', '')
comment = comment[:40]
desc = 'A comment with the string "%s" was found in: "%s".'\
' This could be interesting.'
desc = desc % (comment, response.get_url())
i = Info('HTML comment contains HTML code', desc,
response.id, self.get_name())
i.set_dc(request.get_dc())
i.set_uri(response.get_uri())
i.add_to_highlight(html_in_comment.group(0))
kb.kb.append(self, 'html_comment_hides_html', i)
om.out.information(i.get_desc())
self._already_reported_interesting.add(
(comment, response.get_url()))
def _is_new(self, comment, response):
"""
Make sure that we perform a thread safe check on the self._comments dict,
in order to avoid duplicates.
"""
with self._plugin_lock:
#pylint: disable=E1103
comment_data = self._comments.get(comment, None)
if comment_data is None:
self._comments[comment] = [(response.get_url(), response.id), ]
return True
else:
if response.get_url() not in [x[0] for x in comment_data]:
comment_data.append((response.get_url(), response.id))
self._comments[comment] = comment_data
return True
#pylint: enable=E1103
return False
def end(self):
"""
This method is called when the plugin wont be used anymore.
:return: None
"""
inform = []
for comment in self._comments.iterkeys():
urls_with_this_comment = self._comments[comment]
stick_comment = ' '.join(comment.split())
if len(stick_comment) > 40:
msg = 'A comment with the string "%s..." (and %s more bytes)'\
' was found on these URL(s):'
om.out.information(
msg % (stick_comment[:40], str(len(stick_comment) - 40)))
else:
msg = 'A comment containing "%s" was found on these URL(s):'
om.out.information(msg % (stick_comment))
for url, request_id in urls_with_this_comment:
inform.append('- ' + url +
' (request with id: ' + str(request_id) + ')')
inform.sort()
for i in inform:
om.out.information(i)
self._comments.cleanup()
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class DiskDeque(object):
"""
The base code for this file comes from [0], I've modified it to use a
DiskDict which stores the "self.data" dictionary to disk in order to save
memory.
[0] https://code.activestate.com/recipes/259179/
"""
def __init__(self, iterable=(), maxsize=-1):
if not hasattr(self, 'data'):
self.left = self.right = 0
self.data = DiskDict(table_prefix='deque')
self.maxsize = maxsize
self.extend(iterable)
def append(self, x):
self.data[self.right] = x
self.right += 1
if self.maxsize != -1 and len(self) > self.maxsize:
self.popleft()
def appendleft(self, x):
self.left -= 1
self.data[self.left] = x
if self.maxsize != -1 and len(self) > self.maxsize:
self.pop()
def pop(self):
if self.left == self.right:
raise IndexError('cannot pop from empty deque')
self.right -= 1
elem = self.data[self.right]
del self.data[self.right]
return elem
def popleft(self):
if self.left == self.right:
raise IndexError('cannot pop from empty deque')
elem = self.data[self.left]
del self.data[self.left]
self.left += 1
return elem
def clear(self):
self.data.cleanup()
self.left = self.right = 0
def extend(self, iterable):
for elem in iterable:
self.append(elem)
def extendleft(self, iterable):
for elem in iterable:
self.appendleft(elem)
def rotate(self, n=1):
if self:
n %= len(self)
for i in xrange(n):
self.appendleft(self.pop())
def __getitem__(self, i):
if i < 0:
i += len(self)
try:
return self.data[i + self.left]
except KeyError:
raise IndexError
def __setitem__(self, i, value):
if i < 0:
i += len(self)
try:
self.data[i + self.left] = value
except KeyError:
raise IndexError
def __delitem__(self, i):
size = len(self)
if not (-size <= i < size):
raise IndexError
data = self.data
if i < 0:
i += size
for j in xrange(self.left+i, self.right-1):
data[j] = data[j+1]
self.pop()
def __len__(self):
return self.right - self.left
def __cmp__(self, other):
if type(self) != type(other):
return cmp(type(self), type(other))
return cmp(list(self), list(other))
def __repr__(self, _track=[]):
if id(self) in _track:
return '...'
_track.append(id(self))
r = 'deque(%r)' % (list(self),)
_track.remove(id(self))
return r
def __getstate__(self):
return tuple(self)
def __setstate__(self, s):
self.__init__(s[0])
def __hash__(self):
raise TypeError
def __copy__(self):
return self.__class__(self)
def __deepcopy__(self, memo={}):
from copy import deepcopy
result = self.__class__()
memo[id(self)] = result
result.__init__(deepcopy(tuple(self), memo))
return result
class DiskDeque(object):
"""
The base code for this file comes from [0], I've modified it to use a
DiskDict which stores the "self.data" dictionary to disk in order to save
memory.
[0] https://code.activestate.com/recipes/259179/
"""
def __init__(self, iterable=(), maxsize=-1):
if not hasattr(self, 'data'):
self.left = self.right = 0
self.data = DiskDict(table_prefix='deque')
self.maxsize = maxsize
self.extend(iterable)
def append(self, x):
self.data[self.right] = x
self.right += 1
if self.maxsize != -1 and len(self) > self.maxsize:
self.popleft()
def appendleft(self, x):
self.left -= 1
self.data[self.left] = x
if self.maxsize != -1 and len(self) > self.maxsize:
self.pop()
def pop(self):
if self.left == self.right:
raise IndexError('cannot pop from empty deque')
self.right -= 1
elem = self.data[self.right]
del self.data[self.right]
return elem
def popleft(self):
if self.left == self.right:
raise IndexError('cannot pop from empty deque')
elem = self.data[self.left]
del self.data[self.left]
self.left += 1
return elem
def clear(self):
self.data.cleanup()
self.left = self.right = 0
def extend(self, iterable):
for elem in iterable:
self.append(elem)
def extendleft(self, iterable):
for elem in iterable:
self.appendleft(elem)
def rotate(self, n=1):
if self:
n %= len(self)
for i in xrange(n):
self.appendleft(self.pop())
def __getitem__(self, i):
if i < 0:
i += len(self)
try:
return self.data[i + self.left]
except KeyError:
raise IndexError
def __setitem__(self, i, value):
if i < 0:
i += len(self)
try:
self.data[i + self.left] = value
except KeyError:
raise IndexError
def __delitem__(self, i):
size = len(self)
if not (-size <= i < size):
raise IndexError
data = self.data
if i < 0:
i += size
for j in xrange(self.left + i, self.right - 1):
data[j] = data[j + 1]
self.pop()
def __len__(self):
return self.right - self.left
def __cmp__(self, other):
if type(self) != type(other):
return cmp(type(self), type(other))
return cmp(list(self), list(other))
def __repr__(self, _track=[]):
if id(self) in _track:
return '...'
_track.append(id(self))
r = 'deque(%r)' % (list(self), )
_track.remove(id(self))
return r
def __getstate__(self):
return tuple(self)
def __setstate__(self, s):
self.__init__(s[0])
def __hash__(self):
raise TypeError
def __copy__(self):
return self.__class__(self)
def __deepcopy__(self, memo={}):
from copy import deepcopy
result = self.__class__()
memo[id(self)] = result
result.__init__(deepcopy(tuple(self), memo))
return result
class CachedDiskDict(object):
"""
This data structure keeps the `max_in_memory` most frequently accessed
keys in memory and stores the rest on disk.
It is ideal for situations where a DiskDict is frequently accessed,
fast read / writes are required, and items can take considerable amounts
of memory.
"""
def __init__(self, max_in_memory=50, table_prefix=None):
"""
:param max_in_memory: The max number of items to keep in memory
"""
assert max_in_memory > 0, 'In-memory items must be > 0'
table_prefix = self._get_table_prefix(table_prefix)
self._max_in_memory = max_in_memory
self._disk_dict = DiskDict(table_prefix=table_prefix)
self._in_memory = dict()
self._access_count = Counter()
def cleanup(self):
self._disk_dict.cleanup()
def _get_table_prefix(self, table_prefix):
if table_prefix is None:
table_prefix = 'cached_disk_dict_%s' % rand_alpha(16)
else:
args = (table_prefix, rand_alpha(16))
table_prefix = 'cached_disk_dict_%s_%s' % args
return table_prefix
def get(self, key, default=-456):
try:
return self[key]
except KeyError:
if default is not -456:
return default
raise KeyError()
def __getitem__(self, key):
try:
value = self._in_memory[key]
except KeyError:
# This will raise KeyError if k is not found, and that is OK
# because we don't need to increase the access count when the
# key doesn't exist
value = self._disk_dict[key]
self._increase_access_count(key)
return value
def _get_keys_for_memory(self):
"""
:return: Generate the names of the keys that should be kept in memory.
For example, if `max_in_memory` is set to 2 and:
_in_memory: {1: None, 2: None}
_access_count: {1: 10, 2: 20, 3: 5}
_disk_dict: {3: None}
Then the method will generate [1, 2].
"""
return [k for k, v in self._access_count.most_common(self._max_in_memory)]
def _increase_access_count(self, key):
self._access_count.update([key])
keys_for_memory = self._get_keys_for_memory()
self._move_key_to_disk_if_needed(keys_for_memory)
self._move_key_to_memory_if_needed(key, keys_for_memory)
def _move_key_to_disk_if_needed(self, keys_for_memory):
"""
Analyzes the current access count for the last accessed key and
checks if any if the keys in memory should be moved to disk.
:param keys_for_memory: The keys that should be in memory
:return: The name of the key that was moved to disk, or None if
all the keys are still in memory.
"""
for key in self._in_memory:
if key in keys_for_memory:
continue
try:
value = self._in_memory.pop(key)
except KeyError:
return
else:
self._disk_dict[key] = value
return key
def _move_key_to_memory_if_needed(self, key, keys_for_memory):
"""
Analyzes the current access count for the last accessed key and
checks if any if the keys in disk should be moved to memory.
:param key: The key that was last accessed
:param keys_for_memory: The keys that should be in memory
:return: The name of the key that was moved to memory, or None if
all the keys are still on disk.
"""
# The key is already in memory, nothing to do here
if key in self._in_memory:
return
# The key must not be in memory, nothing to do here
if key not in keys_for_memory:
return
try:
value = self._disk_dict.pop(key)
except KeyError:
return
else:
self._in_memory[key] = value
return key
def __setitem__(self, key, value):
if key in self._in_memory:
self._in_memory[key] = value
elif len(self._in_memory) < self._max_in_memory:
self._in_memory[key] = value
else:
self._disk_dict[key] = value
self._increase_access_count(key)
def __delitem__(self, key):
try:
del self._in_memory[key]
except KeyError:
# This will raise KeyError if k is not found, and that is OK
# because we don't need to increase the access count when the
# key doesn't exist
del self._disk_dict[key]
try:
del self._access_count[key]
except KeyError:
# Another thread removed this key
pass
def __contains__(self, key):
if key in self._in_memory:
self._increase_access_count(key)
return True
if key in self._disk_dict:
self._increase_access_count(key)
return True
return False
def __iter__(self):
"""
Decided not to increase the access count when iterating through the
items. In most cases the iteration will be performed on all items,
thus increasing the access count +1 for each, which will leave all
access counts +1, forcing no movements between memory and disk.
"""
for key in self._in_memory:
yield key
for key in self._disk_dict:
yield key
def iteritems(self):
for key, value in self._in_memory.iteritems():
yield key, value
for key, value in self._disk_dict.iteritems():
yield key, value
class ssi(AuditPlugin):
"""
Find server side inclusion vulnerabilities.
:author: Andres Riancho ([email protected])
"""
def __init__(self):
AuditPlugin.__init__(self)
# Internal variables
self._expected_res_mutant = DiskDict()
self._freq_list = DiskList()
re_str = '<!--#exec cmd="echo -n (.*?);echo -n (.*?)" -->'
self._extract_results_re = re.compile(re_str)
def audit(self, freq, orig_response):
"""
Tests an URL for server side inclusion vulnerabilities.
:param freq: A FuzzableRequest
"""
# Create the mutants to send right now,
ssi_strings = self._get_ssi_strings()
mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response)
# Used in end() to detect "persistent SSI"
for mut in mutants:
expected_result = self._extract_result_from_payload(
mut.get_mod_value())
self._expected_res_mutant[expected_result] = mut
self._freq_list.append(freq)
# End of persistent SSI setup
self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants,
self._analyze_result)
def _get_ssi_strings(self):
"""
This method returns a list of server sides to try to include.
:return: A string, see above.
"""
yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % (rand_alpha(5),
rand_alpha(5))
# TODO: Add mod_perl ssi injection support
# http://www.sens.buffalo.edu/services/webhosting/advanced/perlssi.shtml
#yield <!--#perl sub="sub {print qq/If you see this, mod_perl is working!/;}" -->
def _extract_result_from_payload(self, payload):
"""
Extract the expected result from the payload we're sending.
"""
match = self._extract_results_re.search(payload)
return match.group(1) + match.group(2)
def _analyze_result(self, mutant, response):
"""
Analyze the result of the previously sent request.
:return: None, save the vuln to the kb.
"""
if self._has_no_bug(mutant):
e_res = self._extract_result_from_payload(mutant.get_mod_value())
if e_res in response and not e_res in mutant.get_original_response_body(
):
desc = 'Server side include (SSI) was found at: %s'
desc = desc % mutant.found_at()
v = Vuln.from_mutant('Server side include vulnerability',
desc, severity.HIGH, response.id,
self.get_name(), mutant)
v.add_to_highlight(e_res)
self.kb_append_uniq(self, 'ssi', v)
def end(self):
"""
This method is called when the plugin wont be used anymore and is used
to find persistent SSI vulnerabilities.
Example where a persistent SSI can be found:
Say you have a "guestbook" (a CGI application that allows visitors
to leave messages for everyone to see) on a server that has SSI
enabled. Most such guestbooks around the Net actually allow visitors
to enter HTML code as part of their comments. Now, what happens if a
malicious visitor decides to do some damage by entering the following:
<!--#exec cmd="ls" -->
If the guestbook CGI program was designed carefully, to strip SSI
commands from the input, then there is no problem. But, if it was not,
there exists the potential for a major headache!
For a working example please see moth VM.
"""
multi_in_inst = multi_in(self._expected_res_mutant.keys())
def filtered_freq_generator(freq_list):
already_tested = ScalableBloomFilter()
for freq in freq_list:
if freq not in already_tested:
already_tested.add(freq)
yield freq
def analyze_persistent(freq, response):
for matched_expected_result in multi_in_inst.query(
response.get_body()):
# We found one of the expected results, now we search the
# self._persistent_data to find which of the mutants sent it
# and create the vulnerability
mutant = self._expected_res_mutant[matched_expected_result]
desc = 'Server side include (SSI) was found at: %s' \
' The result of that injection is shown by browsing'\
' to "%s".'
desc = desc % (mutant.found_at(), freq.get_url())
v = Vuln.from_mutant(
'Persistent server side include vulnerability', desc,
severity.HIGH, response.id, self.get_name(), mutant)
v.add_to_highlight(matched_expected_result)
self.kb_append(self, 'ssi', v)
self._send_mutants_in_threads(self._uri_opener.send_mutant,
filtered_freq_generator(self._freq_list),
analyze_persistent,
cache=False)
self._expected_res_mutant.cleanup()
self._freq_list.cleanup()
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class ssi(AuditPlugin):
"""
Find server side inclusion vulnerabilities.
:author: Andres Riancho ([email protected])
"""
def __init__(self):
AuditPlugin.__init__(self)
# Internal variables
self._expected_mutant_dict = DiskDict(table_prefix='ssi')
self._extract_expected_re = re.compile('[1-9]{5}')
def audit(self, freq, orig_response):
"""
Tests an URL for server side inclusion vulnerabilities.
:param freq: A FuzzableRequest
"""
ssi_strings = []
for string in self._get_ssi_strings():
ssi_strings.append(string)
mutants = create_mutants(freq, ssi_strings, orig_resp=orig_response)
self._send_mutants_in_threads(self._uri_opener.send_mutant, mutants,
self._analyze_result)
def _get_ssi_strings(self):
"""
This method returns a list of server sides to try to include.
:return: A string, see above.
"""
# Generic
# yield '<!--#exec cmd="echo -n %s;echo -n %s" -->' % get_seeds()
# Perl SSI
# yield ('<!--#set var="SEED_A" value="%s" -->'
# '<!--#echo var="SEED_A" -->'
# '<!--#set var="SEED_B" value="%s" -->'
# '<!--#echo var="SEED_B" -->' % get_seeds())
# Smarty
# http://www.smarty.net/docsv2/en/language.function.math.tpl
yield '{math equation="x * y" x=%s y=%s}' % get_seeds()
# Mako
# http://docs.makotemplates.org/en/latest/syntax.html
yield '${%s * %s}' % get_seeds()
# Jinja2 and Twig
# http://jinja.pocoo.org/docs/dev/templates/#math
# http://twig.sensiolabs.org/doc/templates.html
yield '{{%s * %s}}' % get_seeds()
# Generic
yield '{%s * %s}' % get_seeds()
# OGNL (struts2)
#yield '${%%{%s * %s}}' % get_seeds()
def _get_expected_results(self, mutant):
"""
Extracts the potential results from the mutant payload and returns them
in a list.
"""
sent_payload = mutant.get_token_payload()
seed_numbers = self._extract_expected_re.findall(sent_payload)
seed_a = int(seed_numbers[0])
seed_b = int(seed_numbers[1])
return [str(seed_a * seed_b), '%s%s' % (seed_a, seed_b)]
def _analyze_result(self, mutant, response):
"""
Analyze the result of the previously sent request.
:return: None, save the vuln to the kb.
"""
# Store the mutants in order to be able to analyze the persistent case
# later
expected_results = self._get_expected_results(mutant)
for expected_result in expected_results:
self._expected_mutant_dict[expected_result] = mutant
# Now we analyze the "reflected" case
if self._has_bug(mutant):
return
for expected_result in expected_results:
if expected_result not in response:
continue
if expected_result in mutant.get_original_response_body():
continue
desc = 'Server side include (SSI) was found at: %s'
desc %= mutant.found_at()
v = Vuln.from_mutant('Server side include vulnerability',
desc, severity.HIGH, response.id,
self.get_name(), mutant)
v.add_to_highlight(expected_result)
self.kb_append_uniq(self, 'ssi', v)
def end(self):
"""
This method is called when the plugin wont be used anymore and is used
to find persistent SSI vulnerabilities.
Example where a persistent SSI can be found:
Say you have a "guest book" (a CGI application that allows visitors
to leave messages for everyone to see) on a server that has SSI
enabled. Most such guest books around the Net actually allow visitors
to enter HTML code as part of their comments. Now, what happens if a
malicious visitor decides to do some damage by entering the following:
<!--#exec cmd="ls" -->
If the guest book CGI program was designed carefully, to strip SSI
commands from the input, then there is no problem. But, if it was not,
there exists the potential for a major headache!
For a working example please see moth VM.
"""
fuzzable_request_set = kb.kb.get_all_known_fuzzable_requests()
self._send_mutants_in_threads(self._uri_opener.send_mutant,
fuzzable_request_set,
self._analyze_persistent,
cache=False)
self._expected_mutant_dict.cleanup()
def _analyze_persistent(self, freq, response):
"""
Analyze the response of sending each fuzzable request found by the
framework, trying to identify any locations where we might have injected
a payload.
:param freq: The fuzzable request
:param response: The HTTP response
:return: None, vulns are stored in KB
"""
multi_in_inst = multi_in(self._expected_mutant_dict.keys())
for matched_expected_result in multi_in_inst.query(
response.get_body()):
# We found one of the expected results, now we search the
# self._expected_mutant_dict to find which of the mutants sent it
# and create the vulnerability
mutant = self._expected_mutant_dict[matched_expected_result]
desc = ('Server side include (SSI) was found at: %s'
' The result of that injection is shown by browsing'
' to "%s".')
desc %= (mutant.found_at(), freq.get_url())
v = Vuln.from_mutant(
'Persistent server side include vulnerability', desc,
severity.HIGH, response.id, self.get_name(), mutant)
v.add_to_highlight(matched_expected_result)
self.kb_append(self, 'ssi', v)
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
class CachedDiskDict(object):
"""
This data structure keeps the `max_in_memory` most frequently accessed
keys in memory and stores the rest on disk.
It is ideal for situations where a DiskDict is frequently accessed,
fast read / writes are required, and items can take considerable amounts
of memory.
"""
def __init__(self, max_in_memory=50, table_prefix=None):
"""
:param max_in_memory: The max number of items to keep in memory
"""
assert max_in_memory > 0, 'In-memory items must be > 0'
table_prefix = self._get_table_prefix(table_prefix)
self._max_in_memory = max_in_memory
self._disk_dict = DiskDict(table_prefix=table_prefix)
self._in_memory = dict()
self._access_count = dict()
def cleanup(self):
self._disk_dict.cleanup()
def _get_table_prefix(self, table_prefix):
if table_prefix is None:
table_prefix = 'cached_disk_dict_%s' % rand_alpha(16)
else:
args = (table_prefix, rand_alpha(16))
table_prefix = 'cached_disk_dict_%s_%s' % args
return table_prefix
def get(self, key, default=-456):
try:
return self[key]
except KeyError:
if default is not -456:
return default
raise KeyError()
def __getitem__(self, key):
try:
value = self._in_memory[key]
except KeyError:
# This will raise KeyError if k is not found, and that is OK
# because we don't need to increase the access count when the
# key doesn't exist
value = self._disk_dict[key]
self._increase_access_count(key)
return value
def _get_keys_for_memory(self):
"""
:return: Generate the names of the keys that should be kept in memory.
For example, if `max_in_memory` is set to 2 and:
_in_memory: {1: None, 2: None}
_access_count: {1: 10, 2: 20, 3: 5}
_disk_dict: {3: None}
Then the method will generate [1, 2].
"""
items = self._access_count.items()
items.sort(sort_by_value)
iterator = min(self._max_in_memory, len(items))
for i in xrange(iterator):
yield items[i][0]
def _belongs_in_memory(self, key):
"""
:param key: A key
:return: True if the key should be stored in memory
"""
if key in self._get_keys_for_memory():
return True
return False
def _increase_access_count(self, key):
access_count = self._access_count.get(key, 0)
access_count += 1
self._access_count[key] = access_count
self._move_key_to_disk_if_needed(key)
self._move_key_to_memory_if_needed(key)
def _move_key_to_disk_if_needed(self, key):
"""
Analyzes the current access count for the last accessed key and
checks if any if the keys in memory should be moved to disk.
:param key: The key that was last accessed
:return: The name of the key that was moved to disk, or None if
all the keys are still in memory.
"""
for key in self._in_memory.keys():
if not self._belongs_in_memory(key):
try:
value = self._in_memory[key]
except KeyError:
return None
else:
self._disk_dict[key] = value
self._in_memory.pop(key, None)
return key
def _move_key_to_memory_if_needed(self, key):
"""
Analyzes the current access count for the last accessed key and
checks if any if the keys in disk should be moved to memory.
:param key: The key that was last accessed
:return: The name of the key that was moved to memory, or None if
all the keys are still on disk.
"""
key_belongs_in_memory = self._belongs_in_memory(key)
if not key_belongs_in_memory:
return None
try:
value = self._disk_dict[key]
except KeyError:
return None
else:
self._in_memory[key] = value
self._disk_dict.pop(key, None)
return key
def __setitem__(self, key, value):
if len(self._in_memory) < self._max_in_memory:
self._in_memory[key] = value
else:
self._disk_dict[key] = value
self._increase_access_count(key)
def __delitem__(self, key):
try:
del self._in_memory[key]
except KeyError:
# This will raise KeyError if k is not found, and that is OK
# because we don't need to increase the access count when the
# key doesn't exist
del self._disk_dict[key]
try:
del self._access_count[key]
except KeyError:
# Another thread removed this key
pass
def __contains__(self, key):
if key in self._in_memory:
self._increase_access_count(key)
return True
if key in self._disk_dict:
self._increase_access_count(key)
return True
return False
def __iter__(self):
"""
Decided not to increase the access count when iterating through the
items. In most cases the iteration will be performed on all items,
thus increasing the access count +1 for each, which will leave all
access counts +1, forcing no movements between memory and disk.
"""
for key in self._in_memory:
yield key
for key in self._disk_dict:
yield key
