class RequestQueuerTestCase(unittest.TestCase):
def setUp(self):
self.deferred = Deferred()
self.mini_web_server = MiniWebServer()
self.rq = RequestQueuer(max_requests_per_host_per_second=3, max_simultaneous_requests_per_host=5)
def tearDown(self):
return self.mini_web_server.shutdown()
def testRequestQueuerOnSuccess(self):
d = self.rq.getPage("http://127.0.0.1:8080/helloworld", timeout=5)
return d
def testRequestQueuerOnFailure(self):
d = self.rq.getPage("http://0.0.0.0:99", timeout=5)
d.addErrback(self._getPageErrback)
return d
def testHostMaxRequestsPerSecond(self,):
self.failUnlessEqual(
self.rq.getHostMaxRequestsPerSecond("example.com"), 3)
self.rq.setHostMaxRequestsPerSecond("example2.com", 7)
self.failUnlessEqual(
self.rq.getHostMaxRequestsPerSecond("example2.com"), 7)
def testHostMaxSimultaneousRequests(self,):
self.failUnlessEqual(
self.rq.getHostMaxSimultaneousRequests("example.com"), 5)
self.rq.setHostMaxSimultaneousRequests("example2.com", 11)
self.failUnlessEqual(
self.rq.getHostMaxSimultaneousRequests("example2.com"),
11)
def testActive(self):
self.failUnlessEqual(isinstance(self.rq.getActive(), int), True)
def testPending(self):
self.failUnlessEqual(isinstance(self.rq.getPending(), int), True)
def testActiveRequestsByHost(self):
self.failUnlessEqual(isinstance(self.rq.getActiveRequestsByHost(), dict), True)
def testPendingRequestsByHost(self):
self.failUnlessEqual(isinstance(self.rq.getPendingRequestsByHost(), dict), True)
def _getPageErrback(self, error):
return True
class PageGetter(object):
def __init__(self,
cassandra_client,
redis_client,
disable_negative_cache=False,
time_offset=0,
rq=None):
"""
Create an Cassandra based HTTP cache.
**Arguments:**
* *cassandra_client* -- Cassandra client object.
**Keyword arguments:**
* *rq* -- Request Queuer object. (Default ``None``)
"""
self.cassandra_client = cassandra_client
self.redis_client = redis_client
self.disable_negative_cache = disable_negative_cache
self.time_offset = time_offset
if rq is None:
self.rq = RequestQueuer()
else:
self.rq = rq
@inlineCallbacks
def getPage(self,
url,
method='GET',
postdata=None,
headers=None,
agent="HiiSpider",
timeout=5,
cookies=None,
follow_redirect=1,
prioritize=False,
hash_url=None,
cache=0,
content_sha1=None,
confirm_cache_write=False,
check_only_tld=False,
disable_negative_cache=False):
"""
Make a cached HTTP Request.
**Arguments:**
* *url* -- URL for the request.
**Keyword arguments:**
* *method* -- HTTP request method. (Default ``'GET'``)
* *postdata* -- Dictionary of strings to post with the request.
(Default ``None``)
* *headers* -- Dictionary of strings to send as request headers.
(Default ``None``)
* *agent* -- User agent to send with request. (Default
``'HiiSpider'``)
* *timeout* -- Request timeout, in seconds. (Default ``60``)
* *cookies* -- Dictionary of strings to send as request cookies.
(Default ``None``).
* *follow_redirect* -- Boolean switch to follow HTTP redirects.
(Default ``True``)
* *prioritize* -- Move this request to the front of the request
queue. (Default ``False``)
* *hash_url* -- URL string used to indicate a common resource.
Example: "http://digg.com" and "http://www.digg.com" could both
use hash_url, "http://digg.com" (Default ``None``)
* *cache* -- Cache mode. ``1``, immediately return contents of
cache if available. ``0``, check resource, return cache if not
stale. ``-1``, ignore cache. (Default ``0``)
* *content_sha1* -- SHA-1 hash of content. If this matches the
hash of data returned by the resource, raises a
StaleContentException.
* *confirm_cache_write* -- Wait to confirm cache write before returning.
* *check_only_tld* -- for negative cache, check only the top level domain name
* *disable_negative_cache* -- disable negative cache for this request
"""
stats.stats.increment("pg.getpage", 0.05)
start = time.time()
request_kwargs = {
"method":method.upper(),
"postdata":postdata,
"headers":headers,
"agent":agent,
"timeout":timeout,
"cookies":cookies,
"follow_redirect":follow_redirect,
"prioritize":prioritize}
cache = int(cache)
if cache not in [-1,0,1]:
raise Exception("Unknown caching mode.")
if not isinstance(url, str):
url = convertToUTF8(url)
if hash_url is not None and not isinstance(hash_url, str):
hash_url = convertToUTF8(hash_url)
# check negative cache
host = _parse(url)[1]
# if check_only_tld is true then parse the url down to the top level domain
if check_only_tld:
host_split = host.split('.', host.count('.')-1)
host = host_split[len(host_split)-1]
# Create request_hash to serve as a cache key from
# either the URL or user-provided hash_url.
hash_items = [hash_url or url, agent]
if postdata:
hash_items.append(repr(postdata))
if headers and 'Authorization' in headers:
items = headers['Authorization'].split(',')
oauth_headers = [item for item in items
if item.find('oauth_consumer_key') > -1 or
item.find('oauth_token') > -1 or
item.find('oauth_token_secret') > -1]
if oauth_headers:
hash_items.append(repr(oauth_headers))
if cookies:
hash_items.append(repr(cookies))
request_hash = sha1(json.dumps(hash_items)).hexdigest()
data = yield self.rq.getPage(url, **request_kwargs)
if "content-sha1" not in data:
data["content-sha1"] = sha1(data["response"]).hexdigest()
if content_sha1 == data["content-sha1"]:
stats.stats.increment('pg.stalecontent')
raise StaleContentException(content_sha1)
returnValue(data)
class PageGetter(object):
def __init__(self,
cassandra_client,
redis_client,
disable_negative_cache=False,
time_offset=0,
rq=None):
"""
Create an Cassandra based HTTP cache.
**Arguments:**
* *cassandra_client* -- Cassandra client object.
**Keyword arguments:**
* *rq* -- Request Queuer object. (Default ``None``)
"""
self.cassandra_client = cassandra_client
self.redis_client = redis_client
self.disable_negative_cache = disable_negative_cache
self.time_offset = time_offset
if rq is None:
self.rq = RequestQueuer()
else:
self.rq = rq
@inlineCallbacks
def getPage(self,
url,
method='GET',
postdata=None,
headers=None,
agent="HiiSpider",
timeout=5,
cookies=None,
follow_redirect=1,
prioritize=False,
hash_url=None,
cache=0,
content_sha1=None,
confirm_cache_write=False,
check_only_tld=False,
disable_negative_cache=False):
"""
Make a cached HTTP Request.
**Arguments:**
* *url* -- URL for the request.
**Keyword arguments:**
* *method* -- HTTP request method. (Default ``'GET'``)
* *postdata* -- Dictionary of strings to post with the request.
(Default ``None``)
* *headers* -- Dictionary of strings to send as request headers.
(Default ``None``)
* *agent* -- User agent to send with request. (Default
``'HiiSpider'``)
* *timeout* -- Request timeout, in seconds. (Default ``60``)
* *cookies* -- Dictionary of strings to send as request cookies.
(Default ``None``).
* *follow_redirect* -- Boolean switch to follow HTTP redirects.
(Default ``True``)
* *prioritize* -- Move this request to the front of the request
queue. (Default ``False``)
* *hash_url* -- URL string used to indicate a common resource.
Example: "http://digg.com" and "http://www.digg.com" could both
use hash_url, "http://digg.com" (Default ``None``)
* *cache* -- Cache mode. ``1``, immediately return contents of
cache if available. ``0``, check resource, return cache if not
stale. ``-1``, ignore cache. (Default ``0``)
* *content_sha1* -- SHA-1 hash of content. If this matches the
hash of data returned by the resource, raises a
StaleContentException.
* *confirm_cache_write* -- Wait to confirm cache write before returning.
* *check_only_tld* -- for negative cache, check only the top level domain name
* *disable_negative_cache* -- disable negative cache for this request
"""
start = time.time()
request_kwargs = {
"method":method.upper(),
"postdata":postdata,
"headers":headers,
"agent":agent,
"timeout":timeout,
"cookies":cookies,
"follow_redirect":follow_redirect,
"prioritize":prioritize}
cache = int(cache)
if cache not in [-1,0,1]:
raise Exception("Unknown caching mode.")
if not isinstance(url, str):
url = convertToUTF8(url)
if hash_url is not None and not isinstance(hash_url, str):
hash_url = convertToUTF8(hash_url)
# check negative cache
host = _parse(url)[1]
# if check_only_tld is true then parse the url down to the top level domain
if check_only_tld:
host_split = host.split('.', host.count('.')-1)
host = host_split[len(host_split)-1]
# Create request_hash to serve as a cache key from
# either the URL or user-provided hash_url.
hash_items = [hash_url or url, agent]
if postdata:
hash_items.append(repr(postdata))
if headers and 'Authorization' in headers:
items = headers['Authorization'].split(',')
oauth_headers = [item for item in items
if item.find('oauth_consumer_key') > -1 or
item.find('oauth_token') > -1 or
item.find('oauth_token_secret') > -1]
if oauth_headers:
hash_items.append(repr(oauth_headers))
if cookies:
hash_items.append(repr(cookies))
request_hash = sha1(json.dumps(hash_items)).hexdigest()
# if not disable_negative_cache and not self.disable_negative_cache:
# yield self.checkNegativeCache(
# 'negative_cache:%s' % host,
# 'negative_req_cache:%s' % request_hash)
# if request_kwargs["method"] != "GET":
# data = yield self.rq.getPage(url, **request_kwargs)
# else:
# data = yield self._getPage(
# url,
# request_hash,
# request_kwargs,
# cache,
# content_sha1,
# confirm_cache_write,
# host)
# logger.info("Got %s after %s" % (host, time.time() - start))
# # Check for stale contents
data = yield self.rq.getPage(url, **request_kwargs)
if "content-sha1" not in data:
data["content-sha1"] = sha1(data["response"]).hexdigest()
if content_sha1 == data["content-sha1"]:
logger.debug("Raising StaleContentException (4) on %s" % request_hash)
raise StaleContentException(content_sha1)
returnValue(data)
@inlineCallbacks
def checkNegativeCache(self, negative_cache_host_key, negative_req_cache_key):
raw_negative_cache_host = yield self.redis_client.get(negative_cache_host_key)
if raw_negative_cache_host:
try:
negative_cache_host = pickle.loads(str(decompress(raw_negative_cache_host)))
if negative_cache_host['timeout'] > time.time():
# we get quite a lot of these, ~500/sec on occasions
stats.stats.increment('pg.negcache.hit', 0.1)
raise NegativeHostCacheException(str(negative_cache_host['error']))
except NegativeHostCacheException:
raise
except Exception, e:
logger.error('Removing host %s from the negative cache: %s' % (request_hash, e))
stats.stats.increment('pg.negcache.flush')
self.redis_client.delete(negative_cache_host_key)
raw_negative_req_cache_item = yield self.redis_client.get(negative_req_cache_key)
if raw_negative_req_cache_item:
try:
negative_req_cache_item = pickle.loads(str(decompress(raw_negative_req_cache_item)))
if negative_req_cache_item['timeout'] > time.time():
stats.stats.increment('pg.negreqcache.hit')
raise NegativeReqCacheException(str(negative_req_cache_item['error']))
except NegativeHostCacheException:
raise
except Exception, e:
logger.error('Removing item %s from the negative cache: %s' % (negative_req_cache_key, e))
stats.stats.increment('pg.negreqcache.flush', 0.5)
self.redis_client.delete(negative_req_cache_key)
class PageGetter:
negitive_cache = {}
def __init__(self, rq=None):
"""
Create an Cassandra based HTTP cache.
**Arguments:**
* *cassandra_client* -- Cassandra client object.
**Keyword arguments:**
* *rq* -- Request Queuer object. (Default ``None``)
"""
if rq is None:
self.rq = RequestQueuer()
else:
self.rq = rq
def getPage(self,
url,
method='GET',
postdata=None,
headers=None,
agent="HiiSpider",
timeout=60,
cookies=None,
follow_redirect=1,
prioritize=False,
hash_url=None,
cache=0,
content_sha1=None,
confirm_cache_write=False,
check_only_tld=False,
disable_negative_cache=False,
):
"""
Make a cached HTTP Request.
**Arguments:**
* *url* -- URL for the request.
**Keyword arguments:**
* *method* -- HTTP request method. (Default ``'GET'``)
* *postdata* -- Dictionary of strings to post with the request.
(Default ``None``)
* *headers* -- Dictionary of strings to send as request headers.
(Default ``None``)
* *agent* -- User agent to send with request. (Default
``'HiiSpider'``)
* *timeout* -- Request timeout, in seconds. (Default ``60``)
* *cookies* -- Dictionary of strings to send as request cookies.
(Default ``None``).
* *follow_redirect* -- Boolean switch to follow HTTP redirects.
(Default ``True``)
* *prioritize* -- Move this request to the front of the request
queue. (Default ``False``)
* *hash_url* -- URL string used to indicate a common resource.
Example: "http://digg.com" and "http://www.digg.com" could both
use hash_url, "http://digg.com" (Default ``None``)
* *cache* -- Cache mode. ``1``, immediately return contents of
cache if available. ``0``, check resource, return cache if not
stale. ``-1``, ignore cache. (Default ``0``)
* *content_sha1* -- SHA-1 hash of content. If this matches the
hash of data returned by the resource, raises a
StaleContentException.
* *confirm_cache_write* -- Wait to confirm cache write before returning.
"""
request_kwargs = {
"method":method.upper(),
"postdata":postdata,
"headers":headers,
"agent":agent,
"timeout":timeout,
"cookies":cookies,
"follow_redirect":follow_redirect,
"prioritize":prioritize}
cache = int(cache)
cache=0
if cache not in [-1,0,1]:
raise Exception("Unknown caching mode.")
if not isinstance(url, str):
url = convertToUTF8(url)
if hash_url is not None and not isinstance(hash_url, str):
hash_url = convertToUTF8(hash_url)
# check negitive cache
host = _parse(url)[1]
# if check_only_tld is true then parse the url down to the top level domain
if check_only_tld:
host_split = host.split('.', host.count('.')-1)
host = host_split[len(host_split)-1]
if host in self.negitive_cache:
if not self.negitive_cache[host]['timeout'] < time.time():
logger.error('Found %s in negitive cache, raising last known exception' % host)
return self.negitive_cache[host]['error'].raiseException()
# Create request_hash to serve as a cache key from
# either the URL or user-provided hash_url.
if hash_url is None:
request_hash = hashlib.sha1(json.dumps([
url,
agent])).hexdigest()
else:
request_hash = hashlib.sha1(json.dumps([
hash_url,
agent])).hexdigest()
d = self.rq.getPage(url, **request_kwargs)
d.addCallback(self._checkForStaleContent, content_sha1, request_hash, host)
d.addErrback(self._getPageErrback, host)
return d
def _checkForStaleContent(self, data, content_sha1, request_hash, host):
if host in self.negitive_cache:
logger.error('Removing %s from negitive cache' % host)
del self.negitive_cache[host]
if "content-sha1" not in data:
data["content-sha1"] = hashlib.sha1(data["response"]).hexdigest()
if content_sha1 == data["content-sha1"]:
logger.debug("Raising StaleContentException (4) on %s" % request_hash)
raise StaleContentException(content_sha1)
else:
return data
def _getPageErrback(self, error, host):
try:
status = int(error.value.status)
except:
status = 500
if status >= 500:
if not host in self.negitive_cache:
logger.error('Adding %s to negitive cache' % host)
self.negitive_cache[host] = {
'timeout': time.time() + 300,
'retries': 1,
'error': error
}
else:
if self.negitive_cache[host]['retries'] <= 5:
self.negitive_cache[host]['timeout'] = time.time() + 600
self.negitive_cache[host]['retries'] += 1
else:
self.negitive_cache[host]['timeout'] = time.time() + 3600
self.negitive_cache[host]['retries'] += 1
self.negitive_cache[host]['error'] = error
logger.error('Updating negitive cache for host %s which has failed %d times' % (host, self.negitive_cache[host]['retries']))
error.raiseException()
