def enable_cache(self, cache_location=None, in_memory=False): """Enable caching for all future HTTP requests. The cache will be created at the default location if none is provided. If the in_memory parameter is True, the cache will be created in memory instead of on disk. This overrides the cache_location parameter. """ if not self._cache: self._cache = APICache(create_db_in_memory=in_memory, db_location=cache_location) self._urlopen = self._cache.make_request
def setUp(self): """Create a MockUrlOpener and an instance of the APICache using it.""" self.urlopener = MockUrlOpener(self.request_headers) self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener)
class APICacheTests(TestCase): """Test cases for the APICache class.""" content = 'foobar' request_headers = { 'http://high_max_age': { 'Cache-Control': 'max-age=10000' }, 'http://zero_max_age': { 'Cache-Control': 'max-age=0', }, 'http://no_cache_etag': { 'Cache-Control': 'no-cache', 'ETag': 'etag', }, 'http://no_cache': { 'Cache-Control': 'no-cache', }, 'http://no_cache_date': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00', }, 'http://no_store': { 'Cache-Control': 'no-store', }, 'http://must_revalidate': { 'Cache-Control': 'must-revalidate', 'ETag': 'etag' }, 'http://vary': { 'Cache-control': 'max-age=1000', 'Vary': 'User-agent' }, 'http://pragma': { 'Pragma': 'no-cache' }, 'http://expired': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', }, 'http://expires_override': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', 'Cache-Control': 'max-age=10000', }, } def setUp(self): """Create a MockUrlOpener and an instance of the APICache using it.""" self.urlopener = MockUrlOpener(self.request_headers) self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener) def test_cache_control_header_max_age_high(self): """Testing the cache with a high max-age value""" request = Request('http://high_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://high_max_age'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_max_age_zero(self): """Testing the cache with a zero max-age value""" request = Request('http://zero_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag(self): """Testing the cache with the no-cache control and a specified ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag_updated(self): """Testing the cache with the no-cache control and an updated ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) # Pretend the end point has been updated since the last request. self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = ( 'new-etag') second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 3) def test_cache_control_header_nocache_with_last_modfied(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache_date', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_last_modified_updated(self): """Testing the cache with the no-cache control and an updated Last-Modified header """ endpoint = 'http://no_cache_lastmodified_updated' future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) self.urlopener.endpoints[endpoint] = { 'hit_count': 0, 'headers': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00' }, } request = Request(endpoint, method='GET') first_resp = self.cache.make_request(request) self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = ( future_date.strftime(CacheEntry.DATE_FORMAT)) second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count(endpoint), 3) def test_cache_control_header_no_store(self): """Testing the cache with the no-store control""" request = Request('http://no_store', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_must_revalidate(self): """Testing the cache with the must-revalidate control""" request = Request('http://must_revalidate', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://must_revalidate'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header(self): """Testing the cache with the Vary header""" request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header_different_requests(self): """Testing the cache with the Vary header and different requests""" first_request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') second_request = Request('http://vary', headers={'User-agent': 'bar'}, method='GET') first_resp = self.cache.make_request(first_request) second_resp = self.cache.make_request(second_request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_pragma_header(self): """Testing the cache with the Pragma: no-cache header""" request = Request('http://pragma', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_expired(self): """Testing the cache with the Expires header in the past""" request = Request('http://expired', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_future(self): """Testing the cache with the Expires header in the future""" # We generate the future date in the C locale so that it is properly # formatted. locale.setlocale(locale.LC_TIME, str('C')) future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC' locale.resetlocale(locale.LC_TIME) self.urlopener.endpoints['http://expires_future'] = { 'hit_count': 0, 'headers': { 'Expires': future_date, }, } request = Request('http://expires_future', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expires_future'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_overriden_by_max_age(self): """Testing the cache with an Expires header that is overridden""" request = Request('http://expires_override', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://expires_override'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_saving_non_ascii_data(self): """Testing writing to the cache with non-ASCII data""" # "Hello world" in Japanese as unicode characters. hello_world = '\u3053\u3093\u306b\u3061\u306f\u4e16\u754c' entry = CacheEntry(url='http://unicode-example', vary_headers={}, max_age=0, etag='etag', local_date=datetime.datetime.now(), last_modified='Sat, 21 Mar 2015 05:33:22 GMT', mime_type='text/plain', item_mime_type=None, response_body=hello_world.encode('utf-8')) try: self.cache._save_entry(entry) except: self.fail('Could not write binary data to the API cache.') try: self.cache._save_entry(entry) except: self.fail('Could not update binary data in the API cache.')
class APICacheTests(TestCase): """Test cases for the APICache class.""" content = 'foobar' request_headers = { 'http://high_max_age': { 'Cache-Control': 'max-age=10000' }, 'http://zero_max_age': { 'Cache-Control': 'max-age=0', }, 'http://no_cache_etag': { 'Cache-Control': 'no-cache', 'ETag': 'etag', }, 'http://no_cache': { 'Cache-Control': 'no-cache', }, 'http://no_cache_date': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00', }, 'http://no_store': { 'Cache-Control': 'no-store', }, 'http://must_revalidate': { 'Cache-Control': 'must-revalidate', 'ETag': 'etag' }, 'http://vary': { 'Cache-control': 'max-age=1000', 'Vary': 'User-agent' }, 'http://pragma': { 'Pragma': 'no-cache' }, 'http://expired': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', }, 'http://expires_override': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', 'Cache-Control': 'max-age=10000', }, } def setUp(self): """Create a MockUrlOpener and an instance of the APICache using it.""" self.urlopener = MockUrlOpener(self.request_headers) self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener) def test_cache_control_header_max_age_high(self): """Testing the cache with a high max-age value""" request = Request('http://high_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://high_max_age'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_max_age_zero(self): """Testing the cache with a zero max-age value""" request = Request('http://zero_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag(self): """Testing the cache with the no-cache control and a specified ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag_updated(self): """Testing the cache with the no-cache control and an updated ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) # Pretend the end point has been updated since the last request. self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = ( 'new-etag') second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 3) def test_cache_control_header_nocache_with_last_modfied(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache_date', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_last_modified_updated(self): """Testing the cache with the no-cache control and an updated Last-Modified header """ endpoint = 'http://no_cache_lastmodified_updated' future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) self.urlopener.endpoints[endpoint] = { 'hit_count': 0, 'headers': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00' }, } request = Request(endpoint, method='GET') first_resp = self.cache.make_request(request) self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = ( future_date.strftime(CacheEntry.DATE_FORMAT)) second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count(endpoint), 3) def test_cache_control_header_no_store(self): """Testing the cache with the no-store control""" request = Request('http://no_store', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_must_revalidate(self): """Testing the cache with the must-revalidate control""" request = Request('http://must_revalidate', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://must_revalidate'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header(self): """Testing the cache with the Vary header""" request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header_different_requests(self): """Testing the cache with the Vary header and different requests""" first_request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') second_request = Request('http://vary', headers={'User-agent': 'bar'}, method='GET') first_resp = self.cache.make_request(first_request) second_resp = self.cache.make_request(second_request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_pragma_header(self): """Testing the cache with the Pragma: no-cache header""" request = Request('http://pragma', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_expired(self): """Testing the cache with the Expires header in the past""" request = Request('http://expired', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_future(self): """Testing the cache with the Expires header in the future""" # We generate the future date in the C locale so that it is properly # formatted. locale.setlocale(locale.LC_TIME, str('C')) future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC' locale.resetlocale(locale.LC_TIME) self.urlopener.endpoints['http://expires_future'] = { 'hit_count': 0, 'headers': { 'Expires': future_date, }, } request = Request('http://expires_future', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expires_future'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_overriden_by_max_age(self): """Testing the cache with an Expires header that is overridden""" request = Request('http://expires_override', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://expires_override'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_saving_non_ascii_data(self): """Testing writing to the cache with non-ASCII data""" # "Hello world" in Japanese as unicode characters. hello_world = '\u3053\u3093\u306b\u3061\u306f\u4e16\u754c' entry = CacheEntry( url='http://unicode-example', vary_headers={}, max_age=0, etag='etag', local_date=datetime.datetime.now(), last_modified='Sat, 21 Mar 2015 05:33:22 GMT', mime_type='text/plain', item_mime_type=None, response_body=hello_world.encode('utf-8')) try: self.cache._save_entry(entry) except: self.fail('Could not write binary data to the API cache.') try: self.cache._save_entry(entry) except: self.fail('Could not update binary data in the API cache.')
class APICacheTests(TestCase): """Test cases for the APICache class.""" content = 'foobar' request_headers = { 'http://high_max_age': { 'Cache-Control': 'max-age=10000' }, 'http://zero_max_age': { 'Cache-Control': 'max-age=0', }, 'http://no_cache_etag': { 'Cache-Control': 'no-cache', 'ETag': 'etag', }, 'http://no_cache': { 'Cache-Control': 'no-cache', }, 'http://no_cache_date': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00', }, 'http://no_store': { 'Cache-Control': 'no-store', }, 'http://must_revalidate': { 'Cache-Control': 'must-revalidate', 'ETag': 'etag' }, 'http://vary': { 'Cache-control': 'max-age=1000', 'Vary': 'User-agent' }, 'http://pragma': { 'Pragma': 'no-cache' }, 'http://expired': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', }, 'http://expires_override': { 'Expires': 'Thu, 01 Dec 1983 20:00:00 GMT', 'Cache-Control': 'max-age=10000', }, } def setUp(self): """Create a MockUrlOpener and an instance of the APICache using it.""" self.urlopener = MockUrlOpener(self.request_headers) self.cache = APICache(create_db_in_memory=True, urlopen=self.urlopener) def test_cache_control_header_max_age_high(self): """Testing the cache with a high max-age value""" request = Request('http://high_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://high_max_age'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_max_age_zero(self): """Testing the cache with a zero max-age value""" request = Request('http://zero_max_age', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://zero_max_age'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag(self): """Testing the cache with the no-cache control and a specified ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_etag_updated(self): """Testing the cache with the no-cache control and an updated ETag""" request = Request('http://no_cache_etag', method='GET') first_resp = self.cache.make_request(request) # Pretend the end point has been updated since the last request. self.urlopener.endpoints['http://no_cache_etag']['headers']['ETag'] = ( 'new-etag') second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_etag'), 3) def test_cache_control_header_nocache_with_last_modfied(self): """Testing the cache with the no-cache control""" request = Request('http://no_cache_date', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_cache_date'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_nocache_with_last_modified_updated(self): """Testing the cache with the no-cache control and an updated Last-Modified header """ endpoint = 'http://no_cache_lastmodified_updated' future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) self.urlopener.endpoints[endpoint] = { 'hit_count': 0, 'headers': { 'Cache-Control': 'no-cache', 'Last-Modified': '1999-12-31T00:00:00' }, } request = Request(endpoint, method='GET') first_resp = self.cache.make_request(request) self.urlopener.endpoints[endpoint]['headers']['Last-Modified'] = ( future_date.strftime(CacheEntry.DATE_FORMAT)) second_resp = self.cache.make_request(request) third_resp = self.cache.make_request(request) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) self.assertTrue(isinstance(third_resp, CachedHTTPResponse)) self.assertEqual(self.urlopener.get_hit_count(endpoint), 3) def test_cache_control_header_no_store(self): """Testing the cache with the no-store control""" request = Request('http://no_store', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://no_store'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_cache_control_header_must_revalidate(self): """Testing the cache with the must-revalidate control""" request = Request('http://must_revalidate', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://must_revalidate'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header(self): """Testing the cache with the Vary header""" request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_vary_header_different_requests(self): """Testing the cache with the Vary header and different requests""" first_request = Request('http://vary', headers={'User-agent': 'foo'}, method='GET') second_request = Request('http://vary', headers={'User-agent': 'bar'}, method='GET') first_resp = self.cache.make_request(first_request) second_resp = self.cache.make_request(second_request) self.assertEqual(self.urlopener.get_hit_count('http://vary'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_pragma_header(self): """Testing the cache with the Pragma: no-cache header""" request = Request('http://pragma', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://pragma'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_expired(self): """Testing the cache with the Expires header in the past""" request = Request('http://expired', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expired'), 2) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertFalse(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_future(self): """Testing the cache with the Expires header in the future""" # We generate the future date in the C locale so that it is properly # formatted. locale.setlocale(locale.LC_TIME, str('C')) future_date = datetime.datetime.utcnow() + datetime.timedelta(days=1) future_date = future_date.strftime(APICache.EXPIRES_FORMAT) + 'UTC' locale.resetlocale(locale.LC_TIME) self.urlopener.endpoints['http://expires_future'] = { 'hit_count': 0, 'headers': { 'Expires': future_date, }, } request = Request('http://expires_future', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual(self.urlopener.get_hit_count('http://expires_future'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse)) def test_expires_header_overriden_by_max_age(self): """Testing the cache with an Expires header that is overridden""" request = Request('http://expires_override', method='GET') first_resp = self.cache.make_request(request) second_resp = self.cache.make_request(request) self.assertEqual( self.urlopener.get_hit_count('http://expires_override'), 1) self.assertFalse(isinstance(first_resp, CachedHTTPResponse)) self.assertTrue(isinstance(second_resp, CachedHTTPResponse))
def __init__(self, url, cookie_file=None, username=None, password=None, api_token=None, agent=None, session=None, disable_proxy=False, auth_callback=None, otp_token_callback=None): self.url = url if not self.url.endswith('/'): self.url += '/' self.url = self.url + 'api/' self.cookie_jar, self.cookie_file = create_cookie_jar( cookie_file=cookie_file) try: self.cookie_jar.load(ignore_expires=True) except IOError: pass if session: parsed_url = urlparse(url) # Get the cookie domain from the url. If the domain # does not contain a '.' (e.g. 'localhost'), we assume # it is a local domain and suffix it (See RFC 2109). domain = parsed_url[1].partition(':')[0] # Remove Port. if domain.count('.') < 1: domain = '%s.local' % domain cookie = Cookie(version=0, name=RB_COOKIE_NAME, value=session, port=None, port_specified=False, domain=domain, domain_specified=True, domain_initial_dot=True, path=parsed_url[2], path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={'HttpOnly': None}) self.cookie_jar.set_cookie(cookie) self.cookie_jar.save() # Set up the HTTP libraries to support all of the features we need. password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password, api_token, auth_callback, otp_token_callback) self.preset_auth_handler = PresetHTTPAuthHandler( self.url, password_mgr) handlers = [] if disable_proxy: handlers.append(ProxyHandler({})) handlers += [ HTTPCookieProcessor(self.cookie_jar), ReviewBoardHTTPBasicAuthHandler(password_mgr), HTTPDigestAuthHandler(password_mgr), self.preset_auth_handler, ReviewBoardHTTPErrorProcessor(), ] if agent: self.agent = agent else: self.agent = ('RBTools/' + get_package_version()).encode('utf-8') opener = build_opener(*handlers) opener.addheaders = [ (b'User-agent', self.agent), ] install_opener(opener) self._cache = APICache()
class ReviewBoardServer(object): """Represents a Review Board server we are communicating with. Provides methods for executing HTTP requests on a Review Board server's Web API. The ``auth_callback`` parameter can be used to specify a callable which will be called when authentication fails. This callable will be passed the realm, and url of the Review Board server and should return a 2-tuple of username, password. The user can be prompted for their credentials using this mechanism. """ def __init__(self, url, cookie_file=None, username=None, password=None, api_token=None, agent=None, session=None, disable_proxy=False, auth_callback=None, otp_token_callback=None): self.url = url if not self.url.endswith('/'): self.url += '/' self.url = self.url + 'api/' self.cookie_jar, self.cookie_file = create_cookie_jar( cookie_file=cookie_file) try: self.cookie_jar.load(ignore_expires=True) except IOError: pass if session: parsed_url = urlparse(url) # Get the cookie domain from the url. If the domain # does not contain a '.' (e.g. 'localhost'), we assume # it is a local domain and suffix it (See RFC 2109). domain = parsed_url[1].partition(':')[0] # Remove Port. if domain.count('.') < 1: domain = '%s.local' % domain cookie = Cookie(version=0, name=RB_COOKIE_NAME, value=session, port=None, port_specified=False, domain=domain, domain_specified=True, domain_initial_dot=True, path=parsed_url[2], path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest={'HttpOnly': None}) self.cookie_jar.set_cookie(cookie) self.cookie_jar.save() # Set up the HTTP libraries to support all of the features we need. password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password, api_token, auth_callback, otp_token_callback) self.preset_auth_handler = PresetHTTPAuthHandler( self.url, password_mgr) handlers = [] if disable_proxy: handlers.append(ProxyHandler({})) handlers += [ HTTPCookieProcessor(self.cookie_jar), ReviewBoardHTTPBasicAuthHandler(password_mgr), HTTPDigestAuthHandler(password_mgr), self.preset_auth_handler, ReviewBoardHTTPErrorProcessor(), ] if agent: self.agent = agent else: self.agent = ('RBTools/' + get_package_version()).encode('utf-8') opener = build_opener(*handlers) opener.addheaders = [ (b'User-agent', self.agent), ] install_opener(opener) self._cache = APICache() def login(self, username, password): """Reset the user information""" self.preset_auth_handler.reset(username, password) def process_error(self, http_status, data): """Processes an error, raising an APIError with the information.""" try: rsp = json_loads(data) assert rsp['stat'] == 'fail' logging.debug('Got API Error %d (HTTP code %d): %s' % (rsp['err']['code'], http_status, rsp['err']['msg'])) logging.debug('Error data: %r' % rsp) raise create_api_error(http_status, rsp['err']['code'], rsp, rsp['err']['msg']) except ValueError: logging.debug('Got HTTP error: %s: %s' % (http_status, data)) raise APIError(http_status, None, None, data) def make_request(self, request): """Perform an http request. The request argument should be an instance of 'rbtools.api.request.HttpRequest'. """ try: content_type, body = request.encode_multipart_formdata() headers = request.headers if body: headers.update({ b'Content-Type': content_type, b'Content-Length': str(len(body)), }) else: headers[b'Content-Length'] = '0' r = Request(request.url.encode('utf-8'), body, headers, request.method) rsp = self._cache.make_request(r) except HTTPError as e: self.process_error(e.code, e.read()) except URLError as e: raise ServerInterfaceError('%s' % e.reason) try: self.cookie_jar.save() except IOError: pass return rsp
def enable_cache(self): """Enable caching for all future requests.""" if not self._cache: self._cache = APICache() self._urlopen = self._cache.make_request