class TestETag(object): """Test our equal priority caching with ETags Equal Priority Caching is a term I've defined to describe when ETags are cached orthgonally from Time Based Caching. """ @pytest.fixture() def sess(self, server): self.etag_url = urljoin(server.application_url, "/etag") self.update_etag_url = urljoin(server.application_url, "/update_etag") self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache, serializer=NullSerializer()) return sess def test_etags_get_example(self, sess, server): """RFC 2616 14.26 The If-None-Match request-header field is used with a method to make it conditional. A client that has one or more entities previously obtained from the resource can verify that none of those entities is current by including a list of their associated entity tags in the If-None-Match header field. The purpose of this feature is to allow efficient updates of cached information with a minimum amount of transaction overhead If any of the entity tags match the entity tag of the entity that would have been returned in the response to a similar GET request (without the If-None-Match header) on that resource, [...] then the server MUST NOT perform the requested method, [...]. Instead, if the request method was GET or HEAD, the server SHOULD respond with a 304 (Not Modified) response, including the cache-related header fields (particularly ETag) of one of the entities that matched. (Paraphrased) A server may provide an ETag header on a response. On subsequent queries, the client may reference the value of this Etag header in an If-None-Match header; on receiving such a header, the server can check whether the entity at that URL has changed from the clients last version, and if not, it can return a 304 to indicate the client can use it's current representation. """ r = sess.get(self.etag_url) # make sure we cached it assert self.cache.get(self.etag_url) == r.raw # make the same request resp = sess.get(self.etag_url) assert resp.raw == r.raw assert resp.from_cache # tell the server to change the etags of the response sess.get(self.update_etag_url) resp = sess.get(self.etag_url) assert resp != r assert not resp.from_cache # Make sure we updated our cache with the new etag'd response. assert self.cache.get(self.etag_url) == resp.raw
class TestDisabledETags(object): """Test our use of ETags when the response is stale and the response has an ETag. """ @pytest.fixture() def sess(self, server): self.etag_url = urljoin(server.application_url, "/etag") self.update_etag_url = urljoin(server.application_url, "/update_etag") self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False, serializer=NullSerializer()) return sess def test_expired_etags_if_none_match_response(self, sess): """Make sure an expired response that contains an ETag uses the If-None-Match header. """ # get our response r = sess.get(self.etag_url) # expire our request by changing the date. Our test endpoint # doesn't provide time base caching headers, so we add them # here in order to expire the request. r.headers["Date"] = "Tue, 26 Nov 2012 00:50:49 GMT" self.cache.set(self.etag_url, r) r = sess.get(self.etag_url) assert r.from_cache assert "if-none-match" in r.request.headers assert r.status_code == 200
def sess(self, server): self.url = server.application_url self.cache = DictCache() sess = Session() sess.mount( 'http://', CacheControlAdapter(self.cache, serializer=NullSerializer()), ) return sess
def sess(self, server, url): self.etag_url = urljoin(url, '/etag') self.update_etag_url = urljoin(url, '/update_etag') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False, serializer=NullSerializer()) return sess
def sess(self, server): self.etag_url = urljoin(server.application_url, '/etag') self.update_etag_url = urljoin(server.application_url, '/update_etag') self.cache = DictCache() sess = CacheControl( requests.Session(), cache=self.cache, serializer=NullSerializer(), ) return sess
def sess(self, url): self.etag_url = urljoin(url, "/etag") self.update_etag_url = urljoin(url, "/update_etag") self.cache = DictCache() sess = CacheControl( requests.Session(), cache=self.cache, serializer=NullSerializer() ) yield sess # closing session object sess.close()
def test_update_cached_response_with_valid_headers(self): cached_resp = Mock(headers={ 'ETag': 'jfd9094r808', 'Content-Length': 100 }) # Set our content length to 200. That would be a mistake in # the server, but we'll handle it gracefully... for now. resp = Mock(headers={'ETag': '28371947465', 'Content-Length': 200}) cache = DictCache({self.url: cached_resp}) cc = CacheController(cache) # skip our in/out processing cc.serializer = Mock() cc.serializer.loads.return_value = cached_resp cc.cache_url = Mock(return_value='http://foo.com') req_headers = {} request = type('Request', (object, ), { 'headers': req_headers, 'url': 'http://example.com' })() result = cc.update_cached_response(request, resp) assert result.headers['ETag'] == resp.headers['ETag'] assert result.headers['Content-Length'] == 100
def test_cache_request_no_headers(self): cached_resp = Mock( headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200 ) self.c.cache = DictCache({self.url: cached_resp}) resp = self.req({}) assert not resp
def test_cache_request_unfresh_max_age(self): earlier = time.time() - 3700 # epoch - 1h01m40s now = time.strftime(TIME_FMT, time.gmtime(earlier)) resp = Mock(headers={"cache-control": "max-age=3600", "date": now}) self.c.cache = DictCache({self.url: resp}) r = self.req({}) assert not r
class TestMaxAge(object): @pytest.fixture() def sess(self, url): self.url = url self.cache = DictCache() sess = Session() sess.mount( 'http://', CacheControlAdapter(self.cache, serializer=NullSerializer()), ) return sess def test_client_max_age_0(self, sess): """ Making sure when the client uses max-age=0 we don't get a cached copy even though we're still fresh. """ print('first request') r = sess.get(self.url) assert self.cache.get(self.url) == r.raw print('second request') r = sess.get(self.url, headers={'Cache-Control': 'max-age=0'}) # don't remove from the cache assert self.cache.get(self.url) assert not r.from_cache def test_client_max_age_3600(self, sess): """ Verify we get a cached value when the client has a reasonable max-age value. """ r = sess.get(self.url) assert self.cache.get(self.url) == r.raw # request that we don't want a new one unless r = sess.get(self.url, headers={'Cache-Control': 'max-age=3600'}) assert r.from_cache is True # now lets grab one that forces a new request b/c the cache # has expired. To do that we'll inject a new time value. resp = self.cache.get(self.url) resp.headers['date'] = 'Tue, 15 Nov 1994 08:12:31 GMT' r = sess.get(self.url) assert not r.from_cache
def __init__(self, driver, connection_name, connection_config): super(GithubConnection, self).__init__(driver, connection_name, connection_config) self._change_cache = {} self._project_branch_cache = {} self.projects = {} self.git_ssh_key = self.connection_config.get('sshkey') self.server = self.connection_config.get('server', 'github.com') self.canonical_hostname = self.connection_config.get( 'canonical_hostname', self.server) self.source = driver.getSource(self) self.event_queue = queue.Queue() if self.server == 'github.com': self.base_url = GITHUB_BASE_URL else: self.base_url = 'https://%s/api/v3' % self.server # ssl verification must default to true verify_ssl = self.connection_config.get('verify_ssl', 'true') self.verify_ssl = True if verify_ssl.lower() == 'false': self.verify_ssl = False self._github = None self.app_id = None self.app_key = None self.sched = None self.installation_map = {} self.installation_token_cache = {} # NOTE(jamielennox): Better here would be to cache to memcache or file # or something external - but zuul already sucks at restarting so in # memory probably doesn't make this much worse. # NOTE(tobiash): Unlike documented cachecontrol doesn't priorize # the etag caching but doesn't even re-request until max-age was # elapsed. # # Thus we need to add a custom caching heuristic which simply drops # the cache-control header containing max-age. This way we force # cachecontrol to only rely on the etag headers. # # http://cachecontrol.readthedocs.io/en/latest/etags.html # http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html class NoAgeHeuristic(BaseHeuristic): def update_headers(self, response): if 'cache-control' in response.headers: del response.headers['cache-control'] self.cache_adapter = cachecontrol.CacheControlAdapter( DictCache(), cache_etags=True, heuristic=NoAgeHeuristic()) # The regex is based on the connection host. We do not yet support # cross-connection dependency gathering self.depends_on_re = re.compile( r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server, re.MULTILINE | re.IGNORECASE)
def sess(self, server): self.etag_url = urljoin(server.application_url, '/etag') self.update_etag_url = urljoin(server.application_url, '/update_etag') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False) return sess
def test_cached_request_with_bad_max_age_headers_not_returned(self): now = time.strftime(TIME_FMT, time.gmtime()) # Not a valid header; this would be from a misconfigured server resp = Mock(headers={"cache-control": "max-age=xxx", "date": now}) self.c.cache = DictCache({self.url: resp}) assert not self.req({})
def test_cache_request_fresh_max_age(self): now = time.strftime(TIME_FMT, time.gmtime()) resp = Mock(headers={"cache-control": "max-age=3600", "date": now}) cache = DictCache({self.url: resp}) self.c.cache = cache r = self.req({}) assert r == resp
def sess(self, url): self.url = url self.cache = DictCache() sess = Session() sess.mount( "http://", CacheControlAdapter(self.cache, serializer=NullSerializer()) ) return sess
def test_cache_request_no_headers(self): cached_resp = Mock(headers={ 'ETag': 'jfd9094r808', 'Content-Length': 100 }) self.c.cache = DictCache({self.url: cached_resp}) resp = self.req({}) assert not resp
def test_cache_request_fresh_expires(self): later = time.time() + 86400 # GMT + 1 day expires = time.strftime(TIME_FMT, time.gmtime(later)) now = time.strftime(TIME_FMT, time.gmtime()) resp = Mock(headers={"expires": expires, "date": now}, status=200) cache = DictCache({self.url: resp}) self.c.cache = cache r = self.req({}) assert r == resp
def sess(self, chunking_server): self.url = chunking_server.base_url self.cache = DictCache() sess = Session() sess.mount( 'http://', CacheControlAdapter(self.cache, serializer=NullSerializer()), ) return sess
def test_cache_request_unfresh_expires(self): sooner = time.time() - 86400 # GMT - 1 day expires = time.strftime(TIME_FMT, time.gmtime(sooner)) now = time.strftime(TIME_FMT, time.gmtime()) resp = Mock(headers={"expires": expires, "date": now}) cache = DictCache({self.url: resp}) self.c.cache = cache r = self.req({}) assert not r
def get_session(): adapter = CacheControlAdapter( DictCache(), cache_etags=True, serializer=None, heuristic=None ) sess = requests.Session() sess.mount("http://", adapter) sess.mount("https://", adapter) sess.cache_controller = adapter.controller return sess
def add_cache_control(session, cache_control_config): """Add cache_control adapter to session object.""" adapter = CacheControlAdapter( DictCache(), cache_etags=cache_control_config.get('cache_etags', True), serializer=cache_control_config.get('serializer', None), heuristic=cache_control_config.get('heuristic', None), ) session.mount('http://', adapter) session.mount('https://', adapter) session.cache_controller = adapter.controller
def test_cache_response_no_store(self): resp = Mock() cache = DictCache({self.url: resp}) cc = CacheController(cache) cache_url = cc.cache_url(self.url) resp = self.resp({"cache-control": "no-store"}) assert cc.cache.get(cache_url) cc.cache_response(self.req(), resp) assert not cc.cache.get(cache_url)
class TestVary(object): @pytest.fixture() def sess(self, server): self.url = urljoin(server.application_url, '/vary_accept') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache) return sess def test_vary_example(self, sess): """RFC 2616 13.6 When the cache receives a subsequent request whose Request-URI specifies one or more cache entries including a Vary header field, the cache MUST NOT use such a cache entry to construct a response to the new request unless all of the selecting request-headers present in the new request match the corresponding stored request-headers in the original request. Or, in simpler terms, when you make a request and the server returns defines a Vary header, unless all the headers listed in the Vary header are the same, it won't use the cached value. """ r = sess.get(self.url) # make sure we cached it assert self.cache.get(self.url) == r # make the same request resp = sess.get(self.url) assert resp == r assert resp.from_cache # make a similar request, changing the accept header resp = sess.get(self.url, headers={'Accept': 'text/plain, text/html'}) assert resp != r assert not resp.from_cache # Just confirming two things here: # # 1) The server used the vary header # 2) We have more than one header we vary on # # The reason for this is that when we don't specify the header # in the request, it is considered the same in terms of # whether or not to use the cached value. assert 'vary' in r.headers assert len(r.headers['vary'].replace(' ', '').split(',')) == 2
def test_uses_auth_token_to_lookup_response(self): req_headers = {'Authorization': 'Bearer some-token'} resp_headers = {'Vary': 'Authorization'} request = type('Request', (object, ), { 'headers': req_headers, 'url': self.url })() expected_key = CacheController.cache_key(request) response = Mock(headers=resp_headers, status=301) # we set the status to 301 so that it is immediately returned without performing any etag/date checks to see if # cache eviction is required self.c.cache = DictCache({expected_key: response}) assert self.req(headers=req_headers) == response
def test_cache_response_no_store(self): resp = Mock() cache = DictCache({self.url: resp}) cc = CacheController(cache) request = type('Request', (object, ), { 'headers': {}, 'url': self.url })() cache_key = cc.cache_key(request) resp = self.resp({'cache-control': 'no-store'}) assert cc.cache.get(cache_key) cc.cache_response(self.req(), resp) assert not cc.cache.get(cache_key)
def __init__(self): session = requests.Session() if not self.__class__._cache: if self.backend == "RedisCache": pool = redis.ConnectionPool(host=self.redis_host, port=self.redis_port, db=0) r = redis.Redis(connection_pool=pool) self.__class__._cache = RedisCache(r) elif self.backend == "FileCache": self.__class__._cache = FileCache(self.file_cache_path) else: self.__class__._cache = DictCache() session = CacheControl(session, heuristic=DefaultHeuristic(self.expire_after), cache=self.__class__._cache) super(CachedRemoteResource, self).__init__(session)
def test_update_cached_response_with_valid_headers(self): cached_resp = Mock(headers={"ETag": "jfd9094r808", "Content-Length": 100}) # Set our content length to 200. That would be a mistake in # the server, but we'll handle it gracefully... for now. resp = Mock(headers={"ETag": "28371947465", "Content-Length": 200}) cache = DictCache({self.url: cached_resp}) cc = CacheController(cache) # skip our in/out processing cc.serializer = Mock() cc.serializer.loads.return_value = cached_resp cc.cache_url = Mock(return_value="http://foo.com") result = cc.update_cached_response(Mock(), resp) assert result.headers["ETag"] == resp.headers["ETag"] assert result.headers["Content-Length"] == 100
class TestChunkedResponse(object): @pytest.fixture() def sess(self, chunking_server): self.url = chunking_server.base_url self.cache = DictCache() sess = Session() sess.mount( 'http://', CacheControlAdapter(self.cache, serializer=NullSerializer()), ) return sess def test_cache_chunked_response(self, sess): """ Verify that an otherwise cacheable response is cached when the response is chunked. """ r = sess.get(self.url) assert self.cache.get(self.url) == r.raw r = sess.get(self.url, headers={'Cache-Control': 'max-age=3600'}) assert r.from_cache is True
class TestVary(object): @pytest.fixture() def sess(self, url): self.url = urljoin(url, '/vary_accept') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache) return sess def cached_equal(self, cached, resp): # remove any transfer-encoding headers as they don't apply to # a cached value if 'chunked' in resp.raw.headers.get('transfer-encoding', ''): resp.raw.headers.pop('transfer-encoding') checks = [ cached._fp.getvalue() == resp.content, cached.headers == resp.raw.headers, cached.status == resp.raw.status, cached.version == resp.raw.version, cached.reason == resp.raw.reason, cached.strict == resp.raw.strict, cached.decode_content == resp.raw.decode_content, ] print(checks) pprint(dict(cached.headers)) pprint(dict(resp.raw.headers)) return all(checks) def test_vary_example(self, sess): """RFC 2616 13.6 When the cache receives a subsequent request whose Request-URI specifies one or more cache entries including a Vary header field, the cache MUST NOT use such a cache entry to construct a response to the new request unless all of the selecting request-headers present in the new request match the corresponding stored request-headers in the original request. Or, in simpler terms, when you make a request and the server returns defines a Vary header, unless all the headers listed in the Vary header are the same, it won't use the cached value. """ s = sess.adapters["http://"].controller.serializer r = sess.get(self.url) c = s.loads(r.request, self.cache.get(self.url)) # make sure we cached it assert self.cached_equal(c, r) # make the same request resp = sess.get(self.url) assert self.cached_equal(c, resp) assert resp.from_cache # make a similar request, changing the accept header resp = sess.get(self.url, headers={'Accept': 'text/plain, text/html'}) assert not self.cached_equal(c, resp) assert not resp.from_cache # Just confirming two things here: # # 1) The server used the vary header # 2) We have more than one header we vary on # # The reason for this is that when we don't specify the header # in the request, it is considered the same in terms of # whether or not to use the cached value. assert 'vary' in r.headers assert len(r.headers['vary'].replace(' ', '').split(',')) == 2
def CacheControl(sess, cache=None, cache_etags=True): cache = cache or DictCache() adapter = CacheControlAdapter(cache, cache_etags=cache_etags) sess.mount('http://', adapter) return sess
def __init__(self, cache=None, cache_etags=True): self.cache = cache or DictCache() self.cache_etags = cache_etags
def setup(self): self.c = CacheController(DictCache(), serializer=NullSerializer())
def sess(self, url): self.url = urljoin(url, '/vary_accept') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache) return sess
def __init__(self, cache=None, cache_etags=True, *args, **kw): super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = cache or DictCache() self.controller = CacheController(self.cache, cache_etags=cache_etags)
def sess(self, server): self.url = urljoin(server.application_url, '/vary_accept') self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache) return sess
session = requests.Session() if options.CACHE: try: from cachecontrol import CacheControl from cachecontrol.cache import DictCache from cachecontrol.caches import FileCache except ImportError as e: raise ImportError("CacheControl and lockfile need to be installed " "in order to use CACHE and CACHE_STORE options " "in neo4jrestclient. \n" "Please, run $ pip install CacheControl lockfile") if isinstance(options.CACHE_STORE, string_types): cache = FileCache(options.CACHE_STORE) elif isinstance(options.CACHE_STORE, dict): cache = DictCache(options.CACHE_STORE) else: cache = options.CACHE_STORE session = CacheControl(session, cache=cache) class Request(object): """ Create an HTTP request object for HTTP verbs GET, POST, PUT and DELETE. """ def __init__(self, username=None, password=None, key_file=None, cert_file=None,
class TestETag(object): """Test our equal priority caching with ETags Equal Priority Caching is a term I've defined to describe when ETags are cached orthgonally from Time Based Caching. """ @pytest.fixture() def sess(self, url): self.etag_url = urljoin(url, "/etag") self.update_etag_url = urljoin(url, "/update_etag") self.cache = DictCache() sess = CacheControl( requests.Session(), cache=self.cache, serializer=NullSerializer() ) yield sess # closing session object sess.close() def test_etags_get_example(self, sess, server): """RFC 2616 14.26 The If-None-Match request-header field is used with a method to make it conditional. A client that has one or more entities previously obtained from the resource can verify that none of those entities is current by including a list of their associated entity tags in the If-None-Match header field. The purpose of this feature is to allow efficient updates of cached information with a minimum amount of transaction overhead If any of the entity tags match the entity tag of the entity that would have been returned in the response to a similar GET request (without the If-None-Match header) on that resource, [...] then the server MUST NOT perform the requested method, [...]. Instead, if the request method was GET or HEAD, the server SHOULD respond with a 304 (Not Modified) response, including the cache-related header fields (particularly ETag) of one of the entities that matched. (Paraphrased) A server may provide an ETag header on a response. On subsequent queries, the client may reference the value of this Etag header in an If-None-Match header; on receiving such a header, the server can check whether the entity at that URL has changed from the clients last version, and if not, it can return a 304 to indicate the client can use it's current representation. """ r = sess.get(self.etag_url) # make sure we cached it assert self.cache.get(self.etag_url) == r.raw # make the same request resp = sess.get(self.etag_url) assert resp.raw == r.raw assert resp.from_cache # tell the server to change the etags of the response sess.get(self.update_etag_url) resp = sess.get(self.etag_url) assert resp != r assert not resp.from_cache # Make sure we updated our cache with the new etag'd response. assert self.cache.get(self.etag_url) == resp.raw
def sess(self, server): self.etag_url = urljoin(server.application_url, "/etag") self.update_etag_url = urljoin(server.application_url, "/update_etag") self.cache = DictCache() sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False, serializer=NullSerializer()) return sess