class RequestQueuerTestCase(unittest.TestCase): def setUp(self): self.deferred = Deferred() self.mini_web_server = MiniWebServer() self.rq = RequestQueuer(max_requests_per_host_per_second=3, max_simultaneous_requests_per_host=5) def tearDown(self): return self.mini_web_server.shutdown() def testRequestQueuerOnSuccess(self): d = self.rq.getPage("http://127.0.0.1:8080/helloworld", timeout=5) return d def testRequestQueuerOnFailure(self): d = self.rq.getPage("http://0.0.0.0:99", timeout=5) d.addErrback(self._getPageErrback) return d def testHostMaxRequestsPerSecond(self,): self.failUnlessEqual( self.rq.getHostMaxRequestsPerSecond("example.com"), 3) self.rq.setHostMaxRequestsPerSecond("example2.com", 7) self.failUnlessEqual( self.rq.getHostMaxRequestsPerSecond("example2.com"), 7) def testHostMaxSimultaneousRequests(self,): self.failUnlessEqual( self.rq.getHostMaxSimultaneousRequests("example.com"), 5) self.rq.setHostMaxSimultaneousRequests("example2.com", 11) self.failUnlessEqual( self.rq.getHostMaxSimultaneousRequests("example2.com"), 11) def testActive(self): self.failUnlessEqual(isinstance(self.rq.getActive(), int), True) def testPending(self): self.failUnlessEqual(isinstance(self.rq.getPending(), int), True) def testActiveRequestsByHost(self): self.failUnlessEqual(isinstance(self.rq.getActiveRequestsByHost(), dict), True) def testPendingRequestsByHost(self): self.failUnlessEqual(isinstance(self.rq.getPendingRequestsByHost(), dict), True) def _getPageErrback(self, error): return True
class PageGetter(object): def __init__(self, cassandra_client, redis_client, disable_negative_cache=False, time_offset=0, rq=None): """ Create an Cassandra based HTTP cache. **Arguments:** * *cassandra_client* -- Cassandra client object. **Keyword arguments:** * *rq* -- Request Queuer object. (Default ``None``) """ self.cassandra_client = cassandra_client self.redis_client = redis_client self.disable_negative_cache = disable_negative_cache self.time_offset = time_offset if rq is None: self.rq = RequestQueuer() else: self.rq = rq @inlineCallbacks def getPage(self, url, method='GET', postdata=None, headers=None, agent="HiiSpider", timeout=5, cookies=None, follow_redirect=1, prioritize=False, hash_url=None, cache=0, content_sha1=None, confirm_cache_write=False, check_only_tld=False, disable_negative_cache=False): """ Make a cached HTTP Request. **Arguments:** * *url* -- URL for the request. **Keyword arguments:** * *method* -- HTTP request method. (Default ``'GET'``) * *postdata* -- Dictionary of strings to post with the request. (Default ``None``) * *headers* -- Dictionary of strings to send as request headers. (Default ``None``) * *agent* -- User agent to send with request. (Default ``'HiiSpider'``) * *timeout* -- Request timeout, in seconds. (Default ``60``) * *cookies* -- Dictionary of strings to send as request cookies. (Default ``None``). * *follow_redirect* -- Boolean switch to follow HTTP redirects. (Default ``True``) * *prioritize* -- Move this request to the front of the request queue. (Default ``False``) * *hash_url* -- URL string used to indicate a common resource. Example: "http://digg.com" and "http://www.digg.com" could both use hash_url, "http://digg.com" (Default ``None``) * *cache* -- Cache mode. ``1``, immediately return contents of cache if available. ``0``, check resource, return cache if not stale. ``-1``, ignore cache. (Default ``0``) * *content_sha1* -- SHA-1 hash of content. If this matches the hash of data returned by the resource, raises a StaleContentException. * *confirm_cache_write* -- Wait to confirm cache write before returning. * *check_only_tld* -- for negative cache, check only the top level domain name * *disable_negative_cache* -- disable negative cache for this request """ stats.stats.increment("pg.getpage", 0.05) start = time.time() request_kwargs = { "method":method.upper(), "postdata":postdata, "headers":headers, "agent":agent, "timeout":timeout, "cookies":cookies, "follow_redirect":follow_redirect, "prioritize":prioritize} cache = int(cache) if cache not in [-1,0,1]: raise Exception("Unknown caching mode.") if not isinstance(url, str): url = convertToUTF8(url) if hash_url is not None and not isinstance(hash_url, str): hash_url = convertToUTF8(hash_url) # check negative cache host = _parse(url)[1] # if check_only_tld is true then parse the url down to the top level domain if check_only_tld: host_split = host.split('.', host.count('.')-1) host = host_split[len(host_split)-1] # Create request_hash to serve as a cache key from # either the URL or user-provided hash_url. hash_items = [hash_url or url, agent] if postdata: hash_items.append(repr(postdata)) if headers and 'Authorization' in headers: items = headers['Authorization'].split(',') oauth_headers = [item for item in items if item.find('oauth_consumer_key') > -1 or item.find('oauth_token') > -1 or item.find('oauth_token_secret') > -1] if oauth_headers: hash_items.append(repr(oauth_headers)) if cookies: hash_items.append(repr(cookies)) request_hash = sha1(json.dumps(hash_items)).hexdigest() data = yield self.rq.getPage(url, **request_kwargs) if "content-sha1" not in data: data["content-sha1"] = sha1(data["response"]).hexdigest() if content_sha1 == data["content-sha1"]: stats.stats.increment('pg.stalecontent') raise StaleContentException(content_sha1) returnValue(data)
class PageGetter(object): def __init__(self, cassandra_client, redis_client, disable_negative_cache=False, time_offset=0, rq=None): """ Create an Cassandra based HTTP cache. **Arguments:** * *cassandra_client* -- Cassandra client object. **Keyword arguments:** * *rq* -- Request Queuer object. (Default ``None``) """ self.cassandra_client = cassandra_client self.redis_client = redis_client self.disable_negative_cache = disable_negative_cache self.time_offset = time_offset if rq is None: self.rq = RequestQueuer() else: self.rq = rq @inlineCallbacks def getPage(self, url, method='GET', postdata=None, headers=None, agent="HiiSpider", timeout=5, cookies=None, follow_redirect=1, prioritize=False, hash_url=None, cache=0, content_sha1=None, confirm_cache_write=False, check_only_tld=False, disable_negative_cache=False): """ Make a cached HTTP Request. **Arguments:** * *url* -- URL for the request. **Keyword arguments:** * *method* -- HTTP request method. (Default ``'GET'``) * *postdata* -- Dictionary of strings to post with the request. (Default ``None``) * *headers* -- Dictionary of strings to send as request headers. (Default ``None``) * *agent* -- User agent to send with request. (Default ``'HiiSpider'``) * *timeout* -- Request timeout, in seconds. (Default ``60``) * *cookies* -- Dictionary of strings to send as request cookies. (Default ``None``). * *follow_redirect* -- Boolean switch to follow HTTP redirects. (Default ``True``) * *prioritize* -- Move this request to the front of the request queue. (Default ``False``) * *hash_url* -- URL string used to indicate a common resource. Example: "http://digg.com" and "http://www.digg.com" could both use hash_url, "http://digg.com" (Default ``None``) * *cache* -- Cache mode. ``1``, immediately return contents of cache if available. ``0``, check resource, return cache if not stale. ``-1``, ignore cache. (Default ``0``) * *content_sha1* -- SHA-1 hash of content. If this matches the hash of data returned by the resource, raises a StaleContentException. * *confirm_cache_write* -- Wait to confirm cache write before returning. * *check_only_tld* -- for negative cache, check only the top level domain name * *disable_negative_cache* -- disable negative cache for this request """ start = time.time() request_kwargs = { "method":method.upper(), "postdata":postdata, "headers":headers, "agent":agent, "timeout":timeout, "cookies":cookies, "follow_redirect":follow_redirect, "prioritize":prioritize} cache = int(cache) if cache not in [-1,0,1]: raise Exception("Unknown caching mode.") if not isinstance(url, str): url = convertToUTF8(url) if hash_url is not None and not isinstance(hash_url, str): hash_url = convertToUTF8(hash_url) # check negative cache host = _parse(url)[1] # if check_only_tld is true then parse the url down to the top level domain if check_only_tld: host_split = host.split('.', host.count('.')-1) host = host_split[len(host_split)-1] # Create request_hash to serve as a cache key from # either the URL or user-provided hash_url. hash_items = [hash_url or url, agent] if postdata: hash_items.append(repr(postdata)) if headers and 'Authorization' in headers: items = headers['Authorization'].split(',') oauth_headers = [item for item in items if item.find('oauth_consumer_key') > -1 or item.find('oauth_token') > -1 or item.find('oauth_token_secret') > -1] if oauth_headers: hash_items.append(repr(oauth_headers)) if cookies: hash_items.append(repr(cookies)) request_hash = sha1(json.dumps(hash_items)).hexdigest() # if not disable_negative_cache and not self.disable_negative_cache: # yield self.checkNegativeCache( # 'negative_cache:%s' % host, # 'negative_req_cache:%s' % request_hash) # if request_kwargs["method"] != "GET": # data = yield self.rq.getPage(url, **request_kwargs) # else: # data = yield self._getPage( # url, # request_hash, # request_kwargs, # cache, # content_sha1, # confirm_cache_write, # host) # logger.info("Got %s after %s" % (host, time.time() - start)) # # Check for stale contents data = yield self.rq.getPage(url, **request_kwargs) if "content-sha1" not in data: data["content-sha1"] = sha1(data["response"]).hexdigest() if content_sha1 == data["content-sha1"]: logger.debug("Raising StaleContentException (4) on %s" % request_hash) raise StaleContentException(content_sha1) returnValue(data) @inlineCallbacks def checkNegativeCache(self, negative_cache_host_key, negative_req_cache_key): raw_negative_cache_host = yield self.redis_client.get(negative_cache_host_key) if raw_negative_cache_host: try: negative_cache_host = pickle.loads(str(decompress(raw_negative_cache_host))) if negative_cache_host['timeout'] > time.time(): # we get quite a lot of these, ~500/sec on occasions stats.stats.increment('pg.negcache.hit', 0.1) raise NegativeHostCacheException(str(negative_cache_host['error'])) except NegativeHostCacheException: raise except Exception, e: logger.error('Removing host %s from the negative cache: %s' % (request_hash, e)) stats.stats.increment('pg.negcache.flush') self.redis_client.delete(negative_cache_host_key) raw_negative_req_cache_item = yield self.redis_client.get(negative_req_cache_key) if raw_negative_req_cache_item: try: negative_req_cache_item = pickle.loads(str(decompress(raw_negative_req_cache_item))) if negative_req_cache_item['timeout'] > time.time(): stats.stats.increment('pg.negreqcache.hit') raise NegativeReqCacheException(str(negative_req_cache_item['error'])) except NegativeHostCacheException: raise except Exception, e: logger.error('Removing item %s from the negative cache: %s' % (negative_req_cache_key, e)) stats.stats.increment('pg.negreqcache.flush', 0.5) self.redis_client.delete(negative_req_cache_key)
class PageGetter: negitive_cache = {} def __init__(self, rq=None): """ Create an Cassandra based HTTP cache. **Arguments:** * *cassandra_client* -- Cassandra client object. **Keyword arguments:** * *rq* -- Request Queuer object. (Default ``None``) """ if rq is None: self.rq = RequestQueuer() else: self.rq = rq def getPage(self, url, method='GET', postdata=None, headers=None, agent="HiiSpider", timeout=60, cookies=None, follow_redirect=1, prioritize=False, hash_url=None, cache=0, content_sha1=None, confirm_cache_write=False, check_only_tld=False, disable_negative_cache=False, ): """ Make a cached HTTP Request. **Arguments:** * *url* -- URL for the request. **Keyword arguments:** * *method* -- HTTP request method. (Default ``'GET'``) * *postdata* -- Dictionary of strings to post with the request. (Default ``None``) * *headers* -- Dictionary of strings to send as request headers. (Default ``None``) * *agent* -- User agent to send with request. (Default ``'HiiSpider'``) * *timeout* -- Request timeout, in seconds. (Default ``60``) * *cookies* -- Dictionary of strings to send as request cookies. (Default ``None``). * *follow_redirect* -- Boolean switch to follow HTTP redirects. (Default ``True``) * *prioritize* -- Move this request to the front of the request queue. (Default ``False``) * *hash_url* -- URL string used to indicate a common resource. Example: "http://digg.com" and "http://www.digg.com" could both use hash_url, "http://digg.com" (Default ``None``) * *cache* -- Cache mode. ``1``, immediately return contents of cache if available. ``0``, check resource, return cache if not stale. ``-1``, ignore cache. (Default ``0``) * *content_sha1* -- SHA-1 hash of content. If this matches the hash of data returned by the resource, raises a StaleContentException. * *confirm_cache_write* -- Wait to confirm cache write before returning. """ request_kwargs = { "method":method.upper(), "postdata":postdata, "headers":headers, "agent":agent, "timeout":timeout, "cookies":cookies, "follow_redirect":follow_redirect, "prioritize":prioritize} cache = int(cache) cache=0 if cache not in [-1,0,1]: raise Exception("Unknown caching mode.") if not isinstance(url, str): url = convertToUTF8(url) if hash_url is not None and not isinstance(hash_url, str): hash_url = convertToUTF8(hash_url) # check negitive cache host = _parse(url)[1] # if check_only_tld is true then parse the url down to the top level domain if check_only_tld: host_split = host.split('.', host.count('.')-1) host = host_split[len(host_split)-1] if host in self.negitive_cache: if not self.negitive_cache[host]['timeout'] < time.time(): logger.error('Found %s in negitive cache, raising last known exception' % host) return self.negitive_cache[host]['error'].raiseException() # Create request_hash to serve as a cache key from # either the URL or user-provided hash_url. if hash_url is None: request_hash = hashlib.sha1(json.dumps([ url, agent])).hexdigest() else: request_hash = hashlib.sha1(json.dumps([ hash_url, agent])).hexdigest() d = self.rq.getPage(url, **request_kwargs) d.addCallback(self._checkForStaleContent, content_sha1, request_hash, host) d.addErrback(self._getPageErrback, host) return d def _checkForStaleContent(self, data, content_sha1, request_hash, host): if host in self.negitive_cache: logger.error('Removing %s from negitive cache' % host) del self.negitive_cache[host] if "content-sha1" not in data: data["content-sha1"] = hashlib.sha1(data["response"]).hexdigest() if content_sha1 == data["content-sha1"]: logger.debug("Raising StaleContentException (4) on %s" % request_hash) raise StaleContentException(content_sha1) else: return data def _getPageErrback(self, error, host): try: status = int(error.value.status) except: status = 500 if status >= 500: if not host in self.negitive_cache: logger.error('Adding %s to negitive cache' % host) self.negitive_cache[host] = { 'timeout': time.time() + 300, 'retries': 1, 'error': error } else: if self.negitive_cache[host]['retries'] <= 5: self.negitive_cache[host]['timeout'] = time.time() + 600 self.negitive_cache[host]['retries'] += 1 else: self.negitive_cache[host]['timeout'] = time.time() + 3600 self.negitive_cache[host]['retries'] += 1 self.negitive_cache[host]['error'] = error logger.error('Updating negitive cache for host %s which has failed %d times' % (host, self.negitive_cache[host]['retries'])) error.raiseException()