def __init__(self):
self._cache = LRU(self.LRU_LENGTH)
self._LRULock = threading.RLock()
# These are here for debugging:
self._archive = set()
self._from_LRU = 0.0
self._calculated_more_than_once = 0.0
self._total = 0.0
def test_basic_lru(self):
lru_test = LRU(4)
lru_test['1'] = 1
lru_test['2'] = 1
lru_test['3'] = 1
lru_test['4'] = 1
# Adding one more, the '1' should go away
lru_test['5'] = 1
self.assertNotIn('1', lru_test)
self.assertIn('5', lru_test)
def __init__(self):
#
# Set the opener, I need it to perform some tests and gain
# the knowledge about the server's 404 response bodies.
#
self._uri_opener = None
self._worker_pool = None
#
# Internal variables
#
self._already_analyzed = False
self._404_responses = deque(maxlen=MAX_404_RESPONSES)
self._lock = thread.allocate_lock()
self._fingerprinted_paths = ScalableBloomFilter()
self._directory_uses_404_codes = ScalableBloomFilter()
# It is OK to store 200 here, I'm only storing path+filename as the key,
# and bool as the value.
self.is_404_LRU = LRU(250)
def enable_dns_cache():
"""
DNS cache trick
This will speed up all the test! Before this dns cache voodoo magic every request
to the HTTP server required a DNS query, this is slow on some networks so I added
this feature.
This method was taken from:
# $Id: download.py,v 1.30 2004/05/13 09:55:30 torh Exp $
That is part of :
swup-0.0.20040519/
Developed by:
# Copyright 2001 - 2003 Trustix AS - <http://www.trustix.com>
# Copyright 2003 - 2004 Tor Hveem - <*****@*****.**>
# Copyright 2004 Omar Kilani for tinysofa - <http://www.tinysofa.org>
"""
om.out.debug('Enabling _dns_cache()')
if not hasattr(socket, 'already_configured'):
socket._getaddrinfo = socket.getaddrinfo
_dns_cache = LRU(200)
def _caching_getaddrinfo(*args, **kwargs):
try:
query = (args)
res = _dns_cache[query]
#This was too noisy and not so useful
#om.out.debug('Cached DNS response for domain: ' + query[0] )
return res
except KeyError:
res = socket._getaddrinfo(*args, **kwargs)
_dns_cache[args] = res
om.out.debug('DNS response from DNS server for domain: ' +
query[0])
return res
if not hasattr(socket, 'already_configured'):
socket.getaddrinfo = _caching_getaddrinfo
socket.already_configured = True
def __init__(self, func, lru_size=100):
self.func = func
self.cache = LRU(lru_size)
def test_keyerror(self):
lru_test = LRU(4)
self.assertRaises(KeyError, lru_test.__getitem__, '3')