class TestETag(object):
"""Test our equal priority caching with ETags
Equal Priority Caching is a term I've defined to describe when
ETags are cached orthgonally from Time Based Caching.
"""
@pytest.fixture()
def sess(self, server):
self.etag_url = urljoin(server.application_url, "/etag")
self.update_etag_url = urljoin(server.application_url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache, serializer=NullSerializer())
return sess
def test_etags_get_example(self, sess, server):
"""RFC 2616 14.26
The If-None-Match request-header field is used with a method to make
it conditional. A client that has one or more entities previously
obtained from the resource can verify that none of those entities
is current by including a list of their associated entity tags in
the If-None-Match header field. The purpose of this feature is to
allow efficient updates of cached information with a minimum amount
of transaction overhead
If any of the entity tags match the entity tag of the entity that
would have been returned in the response to a similar GET request
(without the If-None-Match header) on that resource, [...] then
the server MUST NOT perform the requested method, [...]. Instead, if
the request method was GET or HEAD, the server SHOULD respond with
a 304 (Not Modified) response, including the cache-related header
fields (particularly ETag) of one of the entities that matched.
(Paraphrased) A server may provide an ETag header on a response. On
subsequent queries, the client may reference the value of this Etag
header in an If-None-Match header; on receiving such a header, the
server can check whether the entity at that URL has changed from the
clients last version, and if not, it can return a 304 to indicate
the client can use it's current representation.
"""
r = sess.get(self.etag_url)
# make sure we cached it
assert self.cache.get(self.etag_url) == r.raw
# make the same request
resp = sess.get(self.etag_url)
assert resp.raw == r.raw
assert resp.from_cache
# tell the server to change the etags of the response
sess.get(self.update_etag_url)
resp = sess.get(self.etag_url)
assert resp != r
assert not resp.from_cache
# Make sure we updated our cache with the new etag'd response.
assert self.cache.get(self.etag_url) == resp.raw
class TestDisabledETags(object):
"""Test our use of ETags when the response is stale and the
response has an ETag.
"""
@pytest.fixture()
def sess(self, server):
self.etag_url = urljoin(server.application_url, "/etag")
self.update_etag_url = urljoin(server.application_url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False, serializer=NullSerializer())
return sess
def test_expired_etags_if_none_match_response(self, sess):
"""Make sure an expired response that contains an ETag uses
the If-None-Match header.
"""
# get our response
r = sess.get(self.etag_url)
# expire our request by changing the date. Our test endpoint
# doesn't provide time base caching headers, so we add them
# here in order to expire the request.
r.headers["Date"] = "Tue, 26 Nov 2012 00:50:49 GMT"
self.cache.set(self.etag_url, r)
r = sess.get(self.etag_url)
assert r.from_cache
assert "if-none-match" in r.request.headers
assert r.status_code == 200
def sess(self, server):
self.url = server.application_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def sess(self, server, url):
self.etag_url = urljoin(url, '/etag')
self.update_etag_url = urljoin(url, '/update_etag')
self.cache = DictCache()
sess = CacheControl(requests.Session(),
cache=self.cache,
cache_etags=False,
serializer=NullSerializer())
return sess
def sess(self, server):
self.etag_url = urljoin(server.application_url, '/etag')
self.update_etag_url = urljoin(server.application_url, '/update_etag')
self.cache = DictCache()
sess = CacheControl(
requests.Session(),
cache=self.cache,
serializer=NullSerializer(),
)
return sess
def sess(self, url):
self.etag_url = urljoin(url, "/etag")
self.update_etag_url = urljoin(url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(
requests.Session(), cache=self.cache, serializer=NullSerializer()
)
yield sess
# closing session object
sess.close()
def test_update_cached_response_with_valid_headers(self):
cached_resp = Mock(headers={
'ETag': 'jfd9094r808',
'Content-Length': 100
})
# Set our content length to 200. That would be a mistake in
# the server, but we'll handle it gracefully... for now.
resp = Mock(headers={'ETag': '28371947465', 'Content-Length': 200})
cache = DictCache({self.url: cached_resp})
cc = CacheController(cache)
# skip our in/out processing
cc.serializer = Mock()
cc.serializer.loads.return_value = cached_resp
cc.cache_url = Mock(return_value='http://foo.com')
req_headers = {}
request = type('Request', (object, ), {
'headers': req_headers,
'url': 'http://example.com'
})()
result = cc.update_cached_response(request, resp)
assert result.headers['ETag'] == resp.headers['ETag']
assert result.headers['Content-Length'] == 100
def test_cache_request_no_headers(self):
cached_resp = Mock(
headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200
)
self.c.cache = DictCache({self.url: cached_resp})
resp = self.req({})
assert not resp
def test_cache_request_unfresh_max_age(self):
earlier = time.time() - 3700 # epoch - 1h01m40s
now = time.strftime(TIME_FMT, time.gmtime(earlier))
resp = Mock(headers={"cache-control": "max-age=3600", "date": now})
self.c.cache = DictCache({self.url: resp})
r = self.req({})
assert not r
class TestMaxAge(object):
@pytest.fixture()
def sess(self, url):
self.url = url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def test_client_max_age_0(self, sess):
"""
Making sure when the client uses max-age=0 we don't get a
cached copy even though we're still fresh.
"""
print('first request')
r = sess.get(self.url)
assert self.cache.get(self.url) == r.raw
print('second request')
r = sess.get(self.url, headers={'Cache-Control': 'max-age=0'})
# don't remove from the cache
assert self.cache.get(self.url)
assert not r.from_cache
def test_client_max_age_3600(self, sess):
"""
Verify we get a cached value when the client has a
reasonable max-age value.
"""
r = sess.get(self.url)
assert self.cache.get(self.url) == r.raw
# request that we don't want a new one unless
r = sess.get(self.url, headers={'Cache-Control': 'max-age=3600'})
assert r.from_cache is True
# now lets grab one that forces a new request b/c the cache
# has expired. To do that we'll inject a new time value.
resp = self.cache.get(self.url)
resp.headers['date'] = 'Tue, 15 Nov 1994 08:12:31 GMT'
r = sess.get(self.url)
assert not r.from_cache
def __init__(self, driver, connection_name, connection_config):
super(GithubConnection, self).__init__(driver, connection_name,
connection_config)
self._change_cache = {}
self._project_branch_cache = {}
self.projects = {}
self.git_ssh_key = self.connection_config.get('sshkey')
self.server = self.connection_config.get('server', 'github.com')
self.canonical_hostname = self.connection_config.get(
'canonical_hostname', self.server)
self.source = driver.getSource(self)
self.event_queue = queue.Queue()
if self.server == 'github.com':
self.base_url = GITHUB_BASE_URL
else:
self.base_url = 'https://%s/api/v3' % self.server
# ssl verification must default to true
verify_ssl = self.connection_config.get('verify_ssl', 'true')
self.verify_ssl = True
if verify_ssl.lower() == 'false':
self.verify_ssl = False
self._github = None
self.app_id = None
self.app_key = None
self.sched = None
self.installation_map = {}
self.installation_token_cache = {}
# NOTE(jamielennox): Better here would be to cache to memcache or file
# or something external - but zuul already sucks at restarting so in
# memory probably doesn't make this much worse.
# NOTE(tobiash): Unlike documented cachecontrol doesn't priorize
# the etag caching but doesn't even re-request until max-age was
# elapsed.
#
# Thus we need to add a custom caching heuristic which simply drops
# the cache-control header containing max-age. This way we force
# cachecontrol to only rely on the etag headers.
#
# http://cachecontrol.readthedocs.io/en/latest/etags.html
# http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html
class NoAgeHeuristic(BaseHeuristic):
def update_headers(self, response):
if 'cache-control' in response.headers:
del response.headers['cache-control']
self.cache_adapter = cachecontrol.CacheControlAdapter(
DictCache(), cache_etags=True, heuristic=NoAgeHeuristic())
# The regex is based on the connection host. We do not yet support
# cross-connection dependency gathering
self.depends_on_re = re.compile(
r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server,
re.MULTILINE | re.IGNORECASE)
def sess(self, server):
self.etag_url = urljoin(server.application_url, '/etag')
self.update_etag_url = urljoin(server.application_url, '/update_etag')
self.cache = DictCache()
sess = CacheControl(requests.Session(),
cache=self.cache,
cache_etags=False)
return sess
def test_cached_request_with_bad_max_age_headers_not_returned(self):
now = time.strftime(TIME_FMT, time.gmtime())
# Not a valid header; this would be from a misconfigured server
resp = Mock(headers={"cache-control": "max-age=xxx", "date": now})
self.c.cache = DictCache({self.url: resp})
assert not self.req({})
def test_cache_request_fresh_max_age(self):
now = time.strftime(TIME_FMT, time.gmtime())
resp = Mock(headers={"cache-control": "max-age=3600", "date": now})
cache = DictCache({self.url: resp})
self.c.cache = cache
r = self.req({})
assert r == resp
def sess(self, url):
self.url = url
self.cache = DictCache()
sess = Session()
sess.mount(
"http://", CacheControlAdapter(self.cache, serializer=NullSerializer())
)
return sess
def test_cache_request_no_headers(self):
cached_resp = Mock(headers={
'ETag': 'jfd9094r808',
'Content-Length': 100
})
self.c.cache = DictCache({self.url: cached_resp})
resp = self.req({})
assert not resp
def test_cache_request_fresh_expires(self):
later = time.time() + 86400 # GMT + 1 day
expires = time.strftime(TIME_FMT, time.gmtime(later))
now = time.strftime(TIME_FMT, time.gmtime())
resp = Mock(headers={"expires": expires, "date": now}, status=200)
cache = DictCache({self.url: resp})
self.c.cache = cache
r = self.req({})
assert r == resp
def sess(self, chunking_server):
self.url = chunking_server.base_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def sess(self, server, url):
self.etag_url = urljoin(url, '/etag')
self.update_etag_url = urljoin(url, '/update_etag')
self.cache = DictCache()
sess = CacheControl(requests.Session(),
cache=self.cache,
cache_etags=False,
serializer=NullSerializer())
return sess
def test_cache_request_unfresh_expires(self):
sooner = time.time() - 86400 # GMT - 1 day
expires = time.strftime(TIME_FMT, time.gmtime(sooner))
now = time.strftime(TIME_FMT, time.gmtime())
resp = Mock(headers={"expires": expires, "date": now})
cache = DictCache({self.url: resp})
self.c.cache = cache
r = self.req({})
assert not r
def get_session():
adapter = CacheControlAdapter(
DictCache(), cache_etags=True, serializer=None, heuristic=None
)
sess = requests.Session()
sess.mount("http://", adapter)
sess.mount("https://", adapter)
sess.cache_controller = adapter.controller
return sess
def sess(self, url):
self.etag_url = urljoin(url, "/etag")
self.update_etag_url = urljoin(url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(
requests.Session(), cache=self.cache, serializer=NullSerializer()
)
yield sess
# closing session object
sess.close()
def add_cache_control(session, cache_control_config):
"""Add cache_control adapter to session object."""
adapter = CacheControlAdapter(
DictCache(),
cache_etags=cache_control_config.get('cache_etags', True),
serializer=cache_control_config.get('serializer', None),
heuristic=cache_control_config.get('heuristic', None),
)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.cache_controller = adapter.controller
def test_cache_response_no_store(self):
resp = Mock()
cache = DictCache({self.url: resp})
cc = CacheController(cache)
cache_url = cc.cache_url(self.url)
resp = self.resp({"cache-control": "no-store"})
assert cc.cache.get(cache_url)
cc.cache_response(self.req(), resp)
assert not cc.cache.get(cache_url)
class TestVary(object):
@pytest.fixture()
def sess(self, server):
self.url = urljoin(server.application_url, '/vary_accept')
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_vary_example(self, sess):
"""RFC 2616 13.6
When the cache receives a subsequent request whose Request-URI
specifies one or more cache entries including a Vary header field,
the cache MUST NOT use such a cache entry to construct a response
to the new request unless all of the selecting request-headers
present in the new request match the corresponding stored
request-headers in the original request.
Or, in simpler terms, when you make a request and the server
returns defines a Vary header, unless all the headers listed
in the Vary header are the same, it won't use the cached
value.
"""
r = sess.get(self.url)
# make sure we cached it
assert self.cache.get(self.url) == r
# make the same request
resp = sess.get(self.url)
assert resp == r
assert resp.from_cache
# make a similar request, changing the accept header
resp = sess.get(self.url, headers={'Accept': 'text/plain, text/html'})
assert resp != r
assert not resp.from_cache
# Just confirming two things here:
#
# 1) The server used the vary header
# 2) We have more than one header we vary on
#
# The reason for this is that when we don't specify the header
# in the request, it is considered the same in terms of
# whether or not to use the cached value.
assert 'vary' in r.headers
assert len(r.headers['vary'].replace(' ', '').split(',')) == 2
def test_uses_auth_token_to_lookup_response(self):
req_headers = {'Authorization': 'Bearer some-token'}
resp_headers = {'Vary': 'Authorization'}
request = type('Request', (object, ), {
'headers': req_headers,
'url': self.url
})()
expected_key = CacheController.cache_key(request)
response = Mock(headers=resp_headers, status=301)
# we set the status to 301 so that it is immediately returned without performing any etag/date checks to see if
# cache eviction is required
self.c.cache = DictCache({expected_key: response})
assert self.req(headers=req_headers) == response
def test_cache_response_no_store(self):
resp = Mock()
cache = DictCache({self.url: resp})
cc = CacheController(cache)
request = type('Request', (object, ), {
'headers': {},
'url': self.url
})()
cache_key = cc.cache_key(request)
resp = self.resp({'cache-control': 'no-store'})
assert cc.cache.get(cache_key)
cc.cache_response(self.req(), resp)
assert not cc.cache.get(cache_key)
def __init__(self):
session = requests.Session()
if not self.__class__._cache:
if self.backend == "RedisCache":
pool = redis.ConnectionPool(host=self.redis_host, port=self.redis_port, db=0)
r = redis.Redis(connection_pool=pool)
self.__class__._cache = RedisCache(r)
elif self.backend == "FileCache":
self.__class__._cache = FileCache(self.file_cache_path)
else:
self.__class__._cache = DictCache()
session = CacheControl(session, heuristic=DefaultHeuristic(self.expire_after), cache=self.__class__._cache)
super(CachedRemoteResource, self).__init__(session)
def test_update_cached_response_with_valid_headers(self):
cached_resp = Mock(headers={"ETag": "jfd9094r808", "Content-Length": 100})
# Set our content length to 200. That would be a mistake in
# the server, but we'll handle it gracefully... for now.
resp = Mock(headers={"ETag": "28371947465", "Content-Length": 200})
cache = DictCache({self.url: cached_resp})
cc = CacheController(cache)
# skip our in/out processing
cc.serializer = Mock()
cc.serializer.loads.return_value = cached_resp
cc.cache_url = Mock(return_value="http://foo.com")
result = cc.update_cached_response(Mock(), resp)
assert result.headers["ETag"] == resp.headers["ETag"]
assert result.headers["Content-Length"] == 100
class TestChunkedResponse(object):
@pytest.fixture()
def sess(self, chunking_server):
self.url = chunking_server.base_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def test_cache_chunked_response(self, sess):
"""
Verify that an otherwise cacheable response is cached when the response
is chunked.
"""
r = sess.get(self.url)
assert self.cache.get(self.url) == r.raw
r = sess.get(self.url, headers={'Cache-Control': 'max-age=3600'})
assert r.from_cache is True
class TestChunkedResponse(object):
@pytest.fixture()
def sess(self, chunking_server):
self.url = chunking_server.base_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def test_cache_chunked_response(self, sess):
"""
Verify that an otherwise cacheable response is cached when the response
is chunked.
"""
r = sess.get(self.url)
assert self.cache.get(self.url) == r.raw
r = sess.get(self.url, headers={'Cache-Control': 'max-age=3600'})
assert r.from_cache is True
class TestVary(object):
@pytest.fixture()
def sess(self, url):
self.url = urljoin(url, '/vary_accept')
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def cached_equal(self, cached, resp):
# remove any transfer-encoding headers as they don't apply to
# a cached value
if 'chunked' in resp.raw.headers.get('transfer-encoding', ''):
resp.raw.headers.pop('transfer-encoding')
checks = [
cached._fp.getvalue() == resp.content,
cached.headers == resp.raw.headers,
cached.status == resp.raw.status,
cached.version == resp.raw.version,
cached.reason == resp.raw.reason,
cached.strict == resp.raw.strict,
cached.decode_content == resp.raw.decode_content,
]
print(checks)
pprint(dict(cached.headers))
pprint(dict(resp.raw.headers))
return all(checks)
def test_vary_example(self, sess):
"""RFC 2616 13.6
When the cache receives a subsequent request whose Request-URI
specifies one or more cache entries including a Vary header field,
the cache MUST NOT use such a cache entry to construct a response
to the new request unless all of the selecting request-headers
present in the new request match the corresponding stored
request-headers in the original request.
Or, in simpler terms, when you make a request and the server
returns defines a Vary header, unless all the headers listed
in the Vary header are the same, it won't use the cached
value.
"""
s = sess.adapters["http://"].controller.serializer
r = sess.get(self.url)
c = s.loads(r.request, self.cache.get(self.url))
# make sure we cached it
assert self.cached_equal(c, r)
# make the same request
resp = sess.get(self.url)
assert self.cached_equal(c, resp)
assert resp.from_cache
# make a similar request, changing the accept header
resp = sess.get(self.url, headers={'Accept': 'text/plain, text/html'})
assert not self.cached_equal(c, resp)
assert not resp.from_cache
# Just confirming two things here:
#
# 1) The server used the vary header
# 2) We have more than one header we vary on
#
# The reason for this is that when we don't specify the header
# in the request, it is considered the same in terms of
# whether or not to use the cached value.
assert 'vary' in r.headers
assert len(r.headers['vary'].replace(' ', '').split(',')) == 2
def CacheControl(sess, cache=None, cache_etags=True):
cache = cache or DictCache()
adapter = CacheControlAdapter(cache, cache_etags=cache_etags)
sess.mount('http://', adapter)
return sess
def __init__(self, cache=None, cache_etags=True):
self.cache = cache or DictCache()
self.cache_etags = cache_etags
def setup(self):
self.c = CacheController(DictCache(), serializer=NullSerializer())
def sess(self, url):
self.url = urljoin(url, '/vary_accept')
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def __init__(self, cache=None, cache_etags=True, *args, **kw):
super(CacheControlAdapter, self).__init__(*args, **kw)
self.cache = cache or DictCache()
self.controller = CacheController(self.cache, cache_etags=cache_etags)
def sess(self, server):
self.url = urljoin(server.application_url, '/vary_accept')
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
session = requests.Session()
if options.CACHE:
try:
from cachecontrol import CacheControl
from cachecontrol.cache import DictCache
from cachecontrol.caches import FileCache
except ImportError as e:
raise ImportError("CacheControl and lockfile need to be installed "
"in order to use CACHE and CACHE_STORE options "
"in neo4jrestclient. \n"
"Please, run $ pip install CacheControl lockfile")
if isinstance(options.CACHE_STORE, string_types):
cache = FileCache(options.CACHE_STORE)
elif isinstance(options.CACHE_STORE, dict):
cache = DictCache(options.CACHE_STORE)
else:
cache = options.CACHE_STORE
session = CacheControl(session, cache=cache)
class Request(object):
"""
Create an HTTP request object for HTTP
verbs GET, POST, PUT and DELETE.
"""
def __init__(self,
username=None,
password=None,
key_file=None,
cert_file=None,
class TestETag(object):
"""Test our equal priority caching with ETags
Equal Priority Caching is a term I've defined to describe when
ETags are cached orthgonally from Time Based Caching.
"""
@pytest.fixture()
def sess(self, url):
self.etag_url = urljoin(url, "/etag")
self.update_etag_url = urljoin(url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(
requests.Session(), cache=self.cache, serializer=NullSerializer()
)
yield sess
# closing session object
sess.close()
def test_etags_get_example(self, sess, server):
"""RFC 2616 14.26
The If-None-Match request-header field is used with a method to make
it conditional. A client that has one or more entities previously
obtained from the resource can verify that none of those entities
is current by including a list of their associated entity tags in
the If-None-Match header field. The purpose of this feature is to
allow efficient updates of cached information with a minimum amount
of transaction overhead
If any of the entity tags match the entity tag of the entity that
would have been returned in the response to a similar GET request
(without the If-None-Match header) on that resource, [...] then
the server MUST NOT perform the requested method, [...]. Instead, if
the request method was GET or HEAD, the server SHOULD respond with
a 304 (Not Modified) response, including the cache-related header
fields (particularly ETag) of one of the entities that matched.
(Paraphrased) A server may provide an ETag header on a response. On
subsequent queries, the client may reference the value of this Etag
header in an If-None-Match header; on receiving such a header, the
server can check whether the entity at that URL has changed from the
clients last version, and if not, it can return a 304 to indicate
the client can use it's current representation.
"""
r = sess.get(self.etag_url)
# make sure we cached it
assert self.cache.get(self.etag_url) == r.raw
# make the same request
resp = sess.get(self.etag_url)
assert resp.raw == r.raw
assert resp.from_cache
# tell the server to change the etags of the response
sess.get(self.update_etag_url)
resp = sess.get(self.etag_url)
assert resp != r
assert not resp.from_cache
# Make sure we updated our cache with the new etag'd response.
assert self.cache.get(self.etag_url) == resp.raw
def sess(self, server):
self.url = urljoin(server.application_url, '/vary_accept')
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def sess(self, server):
self.etag_url = urljoin(server.application_url, "/etag")
self.update_etag_url = urljoin(server.application_url, "/update_etag")
self.cache = DictCache()
sess = CacheControl(requests.Session(), cache=self.cache, cache_etags=False, serializer=NullSerializer())
return sess
