def enable_cache(self, cache_location=None, in_memory=False):
"""Enable caching for all future HTTP requests.
The cache will be created at the default location if none is provided.
If the in_memory parameter is True, the cache will be created in memory
instead of on disk. This overrides the cache_location parameter.
"""
if not self._cache:
self._cache = APICache(create_db_in_memory=in_memory,
db_location=cache_location)
self._urlopen = self._cache.make_request
def setUp(self):
"""Create a MockUrlOpener and an instance of the APICache using it."""
self.urlopener = MockUrlOpener(self.request_headers)
self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
class APICacheTests(TestCase):
"""Test cases for the APICache class."""
content = 'foobar'
request_headers = {
'http://high_max_age': {
'Cache-Control': 'max-age=10000'
},
'http://zero_max_age': {
'Cache-Control': 'max-age=0',
},
'http://no_cache_etag': {
'Cache-Control': 'no-cache',
'ETag': 'etag',
},
'http://no_cache': {
'Cache-Control': 'no-cache',
},
'http://no_cache_date': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00',
},
'http://no_store': {
'Cache-Control': 'no-store',
},
'http://must_revalidate': {
'Cache-Control': 'must-revalidate',
'ETag': 'etag'
},
'http://vary': {
'Cache-control': 'max-age=1000',
'Vary': 'User-agent'
},
'http://pragma': {
'Pragma': 'no-cache'
},
'http://expired': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
},
'http://expires_override': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
'Cache-Control': 'max-age=10000',
},
}
def setUp(self):
"""Create a MockUrlOpener and an instance of the APICache using it."""
self.urlopener = MockUrlOpener(self.request_headers)
self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
def test_cache_control_header_max_age_high(self):
"""Testing the cache with a high max-age value"""
request = Request('http://high_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://high_max_age'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_max_age_zero(self):
"""Testing the cache with a zero max-age value"""
request = Request('http://zero_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag(self):
"""Testing the cache with the no-cache control and a specified ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag_updated(self):
"""Testing the cache with the no-cache control and an updated ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
# Pretend the end point has been updated since the last request.
self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = (
'new-etag')
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
3)
def test_cache_control_header_nocache_with_last_modfied(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache_date', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_last_modified_updated(self):
"""Testing the cache with the no-cache control and an updated
Last-Modified header
"""
endpoint = 'http://no_cache_lastmodified_updated'
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
self.urlopener.endpoints[endpoint] = {
'hit_count': 0,
'headers': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00'
},
}
request = Request(endpoint, method='GET')
first_resp = self.cache.make_request(request)
self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = (
future_date.strftime(CacheEntry.DATE_FORMAT))
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count(endpoint), 3)
def test_cache_control_header_no_store(self):
"""Testing the cache with the no-store control"""
request = Request('http://no_store', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_must_revalidate(self):
"""Testing the cache with the must-revalidate control"""
request = Request('http://must_revalidate', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://must_revalidate'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header(self):
"""Testing the cache with the Vary header"""
request = Request('http://vary',
headers={'User-agent': 'foo'},
method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header_different_requests(self):
"""Testing the cache with the Vary header and different requests"""
first_request = Request('http://vary',
headers={'User-agent': 'foo'},
method='GET')
second_request = Request('http://vary',
headers={'User-agent': 'bar'},
method='GET')
first_resp = self.cache.make_request(first_request)
second_resp = self.cache.make_request(second_request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_pragma_header(self):
"""Testing the cache with the Pragma: no-cache header"""
request = Request('http://pragma', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_expired(self):
"""Testing the cache with the Expires header in the past"""
request = Request('http://expired', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_future(self):
"""Testing the cache with the Expires header in the future"""
# We generate the future date in the C locale so that it is properly
# formatted.
locale.setlocale(locale.LC_TIME, str('C'))
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC'
locale.resetlocale(locale.LC_TIME)
self.urlopener.endpoints['http://expires_future'] = {
'hit_count': 0,
'headers': {
'Expires': future_date,
},
}
request = Request('http://expires_future', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expires_future'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_overriden_by_max_age(self):
"""Testing the cache with an Expires header that is overridden"""
request = Request('http://expires_override', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://expires_override'), 1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_saving_non_ascii_data(self):
"""Testing writing to the cache with non-ASCII data"""
# "Hello world" in Japanese as unicode characters.
hello_world = '\u3053\u3093\u306b\u3061\u306f\u4e16\u754c'
entry = CacheEntry(url='http://unicode-example',
vary_headers={},
max_age=0,
etag='etag',
local_date=datetime.datetime.now(),
last_modified='Sat, 21 Mar 2015 05:33:22 GMT',
mime_type='text/plain',
item_mime_type=None,
response_body=hello_world.encode('utf-8'))
try:
self.cache._save_entry(entry)
except:
self.fail('Could not write binary data to the API cache.')
try:
self.cache._save_entry(entry)
except:
self.fail('Could not update binary data in the API cache.')
def setUp(self):
"""Create a MockUrlOpener and an instance of the APICache using it."""
self.urlopener = MockUrlOpener(self.request_headers)
self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
class APICacheTests(TestCase):
"""Test cases for the APICache class."""
content = 'foobar'
request_headers = {
'http://high_max_age': {
'Cache-Control': 'max-age=10000'
},
'http://zero_max_age': {
'Cache-Control': 'max-age=0',
},
'http://no_cache_etag': {
'Cache-Control': 'no-cache',
'ETag': 'etag',
},
'http://no_cache': {
'Cache-Control': 'no-cache',
},
'http://no_cache_date': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00',
},
'http://no_store': {
'Cache-Control': 'no-store',
},
'http://must_revalidate': {
'Cache-Control': 'must-revalidate',
'ETag': 'etag'
},
'http://vary': {
'Cache-control': 'max-age=1000',
'Vary': 'User-agent'
},
'http://pragma': {
'Pragma': 'no-cache'
},
'http://expired': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
},
'http://expires_override': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
'Cache-Control': 'max-age=10000',
},
}
def setUp(self):
"""Create a MockUrlOpener and an instance of the APICache using it."""
self.urlopener = MockUrlOpener(self.request_headers)
self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
def test_cache_control_header_max_age_high(self):
"""Testing the cache with a high max-age value"""
request = Request('http://high_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://high_max_age'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_max_age_zero(self):
"""Testing the cache with a zero max-age value"""
request = Request('http://zero_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag(self):
"""Testing the cache with the no-cache control and a specified ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag_updated(self):
"""Testing the cache with the no-cache control and an updated ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
# Pretend the end point has been updated since the last request.
self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = (
'new-etag')
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
3)
def test_cache_control_header_nocache_with_last_modfied(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache_date', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_last_modified_updated(self):
"""Testing the cache with the no-cache control and an updated
Last-Modified header
"""
endpoint = 'http://no_cache_lastmodified_updated'
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
self.urlopener.endpoints[endpoint] = {
'hit_count': 0,
'headers': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00'
},
}
request = Request(endpoint, method='GET')
first_resp = self.cache.make_request(request)
self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = (
future_date.strftime(CacheEntry.DATE_FORMAT))
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count(endpoint), 3)
def test_cache_control_header_no_store(self):
"""Testing the cache with the no-store control"""
request = Request('http://no_store', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_must_revalidate(self):
"""Testing the cache with the must-revalidate control"""
request = Request('http://must_revalidate', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://must_revalidate'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header(self):
"""Testing the cache with the Vary header"""
request = Request('http://vary', headers={'User-agent': 'foo'},
method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header_different_requests(self):
"""Testing the cache with the Vary header and different requests"""
first_request = Request('http://vary', headers={'User-agent': 'foo'},
method='GET')
second_request = Request('http://vary', headers={'User-agent': 'bar'},
method='GET')
first_resp = self.cache.make_request(first_request)
second_resp = self.cache.make_request(second_request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_pragma_header(self):
"""Testing the cache with the Pragma: no-cache header"""
request = Request('http://pragma', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_expired(self):
"""Testing the cache with the Expires header in the past"""
request = Request('http://expired', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_future(self):
"""Testing the cache with the Expires header in the future"""
# We generate the future date in the C locale so that it is properly
# formatted.
locale.setlocale(locale.LC_TIME, str('C'))
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC'
locale.resetlocale(locale.LC_TIME)
self.urlopener.endpoints['http://expires_future'] = {
'hit_count': 0,
'headers': {
'Expires': future_date,
},
}
request = Request('http://expires_future', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expires_future'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_overriden_by_max_age(self):
"""Testing the cache with an Expires header that is overridden"""
request = Request('http://expires_override', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://expires_override'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_saving_non_ascii_data(self):
"""Testing writing to the cache with non-ASCII data"""
# "Hello world" in Japanese as unicode characters.
hello_world = '\u3053\u3093\u306b\u3061\u306f\u4e16\u754c'
entry = CacheEntry(
url='http://unicode-example',
vary_headers={},
max_age=0,
etag='etag',
local_date=datetime.datetime.now(),
last_modified='Sat, 21 Mar 2015 05:33:22 GMT',
mime_type='text/plain',
item_mime_type=None,
response_body=hello_world.encode('utf-8'))
try:
self.cache._save_entry(entry)
except:
self.fail('Could not write binary data to the API cache.')
try:
self.cache._save_entry(entry)
except:
self.fail('Could not update binary data in the API cache.')
class APICacheTests(TestCase):
"""Test cases for the APICache class."""
content = 'foobar'
request_headers = {
'http://high_max_age': {
'Cache-Control': 'max-age=10000'
},
'http://zero_max_age': {
'Cache-Control': 'max-age=0',
},
'http://no_cache_etag': {
'Cache-Control': 'no-cache',
'ETag': 'etag',
},
'http://no_cache': {
'Cache-Control': 'no-cache',
},
'http://no_cache_date': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00',
},
'http://no_store': {
'Cache-Control': 'no-store',
},
'http://must_revalidate': {
'Cache-Control': 'must-revalidate',
'ETag': 'etag'
},
'http://vary': {
'Cache-control': 'max-age=1000',
'Vary': 'User-agent'
},
'http://pragma': {
'Pragma': 'no-cache'
},
'http://expired': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
},
'http://expires_override': {
'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT',
'Cache-Control': 'max-age=10000',
},
}
def setUp(self):
"""Create a MockUrlOpener and an instance of the APICache using it."""
self.urlopener = MockUrlOpener(self.request_headers)
self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
def test_cache_control_header_max_age_high(self):
"""Testing the cache with a high max-age value"""
request = Request('http://high_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://high_max_age'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_max_age_zero(self):
"""Testing the cache with a zero max-age value"""
request = Request('http://zero_max_age', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag(self):
"""Testing the cache with the no-cache control and a specified ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_etag_updated(self):
"""Testing the cache with the no-cache control and an updated ETag"""
request = Request('http://no_cache_etag', method='GET')
first_resp = self.cache.make_request(request)
# Pretend the end point has been updated since the last request.
self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = (
'new-etag')
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'),
3)
def test_cache_control_header_nocache_with_last_modfied(self):
"""Testing the cache with the no-cache control"""
request = Request('http://no_cache_date', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'),
2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_nocache_with_last_modified_updated(self):
"""Testing the cache with the no-cache control and an updated
Last-Modified header
"""
endpoint = 'http://no_cache_lastmodified_updated'
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
self.urlopener.endpoints[endpoint] = {
'hit_count': 0,
'headers': {
'Cache-Control': 'no-cache',
'Last-Modified': '1999-12-31T00:00:00'
},
}
request = Request(endpoint, method='GET')
first_resp = self.cache.make_request(request)
self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = (
future_date.strftime(CacheEntry.DATE_FORMAT))
second_resp = self.cache.make_request(request)
third_resp = self.cache.make_request(request)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
self.assertTrue(isinstance(third_resp, CachedHTTPResponse))
self.assertEqual(self.urlopener.get_hit_count(endpoint), 3)
def test_cache_control_header_no_store(self):
"""Testing the cache with the no-store control"""
request = Request('http://no_store', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_cache_control_header_must_revalidate(self):
"""Testing the cache with the must-revalidate control"""
request = Request('http://must_revalidate', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://must_revalidate'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header(self):
"""Testing the cache with the Vary header"""
request = Request('http://vary',
headers={'User-agent': 'foo'},
method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_vary_header_different_requests(self):
"""Testing the cache with the Vary header and different requests"""
first_request = Request('http://vary',
headers={'User-agent': 'foo'},
method='GET')
second_request = Request('http://vary',
headers={'User-agent': 'bar'},
method='GET')
first_resp = self.cache.make_request(first_request)
second_resp = self.cache.make_request(second_request)
self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_pragma_header(self):
"""Testing the cache with the Pragma: no-cache header"""
request = Request('http://pragma', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_expired(self):
"""Testing the cache with the Expires header in the past"""
request = Request('http://expired', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertFalse(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_future(self):
"""Testing the cache with the Expires header in the future"""
# We generate the future date in the C locale so that it is properly
# formatted.
locale.setlocale(locale.LC_TIME, str('C'))
future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1)
future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC'
locale.resetlocale(locale.LC_TIME)
self.urlopener.endpoints['http://expires_future'] = {
'hit_count': 0,
'headers': {
'Expires': future_date,
},
}
request = Request('http://expires_future', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(self.urlopener.get_hit_count('http://expires_future'),
1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def test_expires_header_overriden_by_max_age(self):
"""Testing the cache with an Expires header that is overridden"""
request = Request('http://expires_override', method='GET')
first_resp = self.cache.make_request(request)
second_resp = self.cache.make_request(request)
self.assertEqual(
self.urlopener.get_hit_count('http://expires_override'), 1)
self.assertFalse(isinstance(first_resp, CachedHTTPResponse))
self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None):
self.url = url
if not self.url.endswith('/'):
self.url += '/'
self.url = self.url + 'api/'
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
if session:
parsed_url = urlparse(url)
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
domain = parsed_url[1].partition(':')[0] # Remove Port.
if domain.count('.') < 1:
domain = '%s.local' % domain
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
self.cookie_jar.save()
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(b'User-agent', self.agent),
]
install_opener(opener)
self._cache = APICache()
class ReviewBoardServer(object):
"""Represents a Review Board server we are communicating with.
Provides methods for executing HTTP requests on a Review Board
server's Web API.
The ``auth_callback`` parameter can be used to specify a callable
which will be called when authentication fails. This callable will
be passed the realm, and url of the Review Board server and should
return a 2-tuple of username, password. The user can be prompted
for their credentials using this mechanism.
"""
def __init__(self,
url,
cookie_file=None,
username=None,
password=None,
api_token=None,
agent=None,
session=None,
disable_proxy=False,
auth_callback=None,
otp_token_callback=None):
self.url = url
if not self.url.endswith('/'):
self.url += '/'
self.url = self.url + 'api/'
self.cookie_jar, self.cookie_file = create_cookie_jar(
cookie_file=cookie_file)
try:
self.cookie_jar.load(ignore_expires=True)
except IOError:
pass
if session:
parsed_url = urlparse(url)
# Get the cookie domain from the url. If the domain
# does not contain a '.' (e.g. 'localhost'), we assume
# it is a local domain and suffix it (See RFC 2109).
domain = parsed_url[1].partition(':')[0] # Remove Port.
if domain.count('.') < 1:
domain = '%s.local' % domain
cookie = Cookie(version=0,
name=RB_COOKIE_NAME,
value=session,
port=None,
port_specified=False,
domain=domain,
domain_specified=True,
domain_initial_dot=True,
path=parsed_url[2],
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={'HttpOnly': None})
self.cookie_jar.set_cookie(cookie)
self.cookie_jar.save()
# Set up the HTTP libraries to support all of the features we need.
password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
api_token, auth_callback,
otp_token_callback)
self.preset_auth_handler = PresetHTTPAuthHandler(
self.url, password_mgr)
handlers = []
if disable_proxy:
handlers.append(ProxyHandler({}))
handlers += [
HTTPCookieProcessor(self.cookie_jar),
ReviewBoardHTTPBasicAuthHandler(password_mgr),
HTTPDigestAuthHandler(password_mgr),
self.preset_auth_handler,
ReviewBoardHTTPErrorProcessor(),
]
if agent:
self.agent = agent
else:
self.agent = ('RBTools/' + get_package_version()).encode('utf-8')
opener = build_opener(*handlers)
opener.addheaders = [
(b'User-agent', self.agent),
]
install_opener(opener)
self._cache = APICache()
def login(self, username, password):
"""Reset the user information"""
self.preset_auth_handler.reset(username, password)
def process_error(self, http_status, data):
"""Processes an error, raising an APIError with the information."""
try:
rsp = json_loads(data)
assert rsp['stat'] == 'fail'
logging.debug('Got API Error %d (HTTP code %d): %s' %
(rsp['err']['code'], http_status, rsp['err']['msg']))
logging.debug('Error data: %r' % rsp)
raise create_api_error(http_status, rsp['err']['code'], rsp,
rsp['err']['msg'])
except ValueError:
logging.debug('Got HTTP error: %s: %s' % (http_status, data))
raise APIError(http_status, None, None, data)
def make_request(self, request):
"""Perform an http request.
The request argument should be an instance of
'rbtools.api.request.HttpRequest'.
"""
try:
content_type, body = request.encode_multipart_formdata()
headers = request.headers
if body:
headers.update({
b'Content-Type': content_type,
b'Content-Length': str(len(body)),
})
else:
headers[b'Content-Length'] = '0'
r = Request(request.url.encode('utf-8'), body, headers,
request.method)
rsp = self._cache.make_request(r)
except HTTPError as e:
self.process_error(e.code, e.read())
except URLError as e:
raise ServerInterfaceError('%s' % e.reason)
try:
self.cookie_jar.save()
except IOError:
pass
return rsp
def enable_cache(self):
"""Enable caching for all future requests."""
if not self._cache:
self._cache = APICache()
self._urlopen = self._cache.make_request
