def test_cache_memoize_iter_compressed(self):
"""Testing cache_memoize_iter with compression"""
cache_key = 'abc123'
data_yielded = []
# This takes into account the size of the pickle data, and will
# get us to exactly 2 chunks of data in cache, each.
data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2)
data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2)
def cache_func():
data_yielded.append('data1')
yield data1
data_yielded.append('data2')
yield data2
self.spy_on(cache_func, call_original=True)
result = cache_memoize_iter(cache_key,
cache_func,
compress_large_data=True)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(data_yielded, [])
self.assertEqual(next(result), data1)
self.assertEqual(data_yielded, ['data1'])
self.assertEqual(next(result), data2)
self.assertEqual(data_yielded, ['data1', 'data2'])
with self.assertRaises(StopIteration):
next(result)
self.assertTrue(cache_func.spy.called)
cache_key_main = make_cache_key(cache_key)
cache_key_0 = make_cache_key('%s-0' % cache_key)
self.assertTrue(cache_key_main in cache)
self.assertTrue(cache_key_0 in cache)
self.assertFalse(make_cache_key('%s-1' % cache_key) in cache)
# Verify the contents of the stored data.
self.assertEqual(cache.get(cache_key_main), '1')
self.assertEqual(
cache.get(cache_key_0)[0],
zlib.compress(pickled_data_1 + pickled_data_2))
# Try fetching the data we stored.
cache_func.spy.reset_calls()
data_yielded = []
result = cache_memoize_iter(cache_key,
cache_func,
compress_large_data=True)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(next(result), data1)
self.assertEqual(next(result), data2)
with self.assertRaises(StopIteration):
next(result)
self.assertEqual(data_yielded, [])
self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_compressed(self):
"""Testing cache_memoize_iter with compression"""
cache_key = 'abc123'
data_yielded = []
# This takes into account the size of the pickle data, and will
# get us to exactly 2 chunks of data in cache, each.
data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2)
data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2)
def cache_func():
data_yielded.append('data1')
yield data1
data_yielded.append('data2')
yield data2
self.spy_on(cache_func, call_original=True)
result = cache_memoize_iter(cache_key, cache_func,
compress_large_data=True)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(data_yielded, [])
self.assertEqual(next(result), data1)
self.assertEqual(data_yielded, ['data1'])
self.assertEqual(next(result), data2)
self.assertEqual(data_yielded, ['data1', 'data2'])
with self.assertRaises(StopIteration):
next(result)
self.assertTrue(cache_func.spy.called)
cache_key_main = make_cache_key(cache_key)
cache_key_0 = make_cache_key('%s-0' % cache_key)
self.assertTrue(cache_key_main in cache)
self.assertTrue(cache_key_0 in cache)
self.assertFalse(make_cache_key('%s-1' % cache_key) in cache)
# Verify the contents of the stored data.
self.assertEqual(cache.get(cache_key_main), '1')
self.assertEqual(cache.get(cache_key_0)[0],
zlib.compress(pickled_data_1 + pickled_data_2))
# Try fetching the data we stored.
cache_func.spy.reset_calls()
data_yielded = []
result = cache_memoize_iter(cache_key, cache_func,
compress_large_data=True)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(next(result), data1)
self.assertEqual(next(result), data2)
with self.assertRaises(StopIteration):
next(result)
self.assertEqual(data_yielded, [])
self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_uncompressed(self):
"""Testing cache_memoize_iter without compression"""
cache_key = 'abc123'
data_yielded = []
data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2)
data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2)
def cache_func():
data_yielded.append('data1')
yield data1
data_yielded.append('data2')
yield data2
self.spy_on(cache_func, call_original=True)
result = cache_memoize_iter(cache_key,
cache_func,
compress_large_data=False)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(data_yielded, [])
self.assertEqual(next(result), data1)
self.assertEqual(data_yielded, ['data1'])
self.assertEqual(next(result), data2)
self.assertEqual(data_yielded, ['data1', 'data2'])
with self.assertRaises(StopIteration):
next(result)
self.assertTrue(cache_func.spy.called)
cache_key_main = make_cache_key(cache_key)
cache_key_0 = make_cache_key('%s-0' % cache_key)
cache_key_1 = make_cache_key('%s-1' % cache_key)
cache_key_2 = make_cache_key('%s-2' % cache_key)
cache_key_3 = make_cache_key('%s-3' % cache_key)
self.assertTrue(cache_key_main in cache)
self.assertTrue(cache_key_0 in cache)
self.assertTrue(cache_key_1 in cache)
self.assertTrue(cache_key_2 in cache)
self.assertTrue(cache_key_3 in cache)
self.assertFalse(make_cache_key('%s-4' % cache_key) in cache)
# Verify the contents of the stored data.
stored_data = b''.join(
cache.get(cache_key_0) + cache.get(cache_key_1) +
cache.get(cache_key_2) + cache.get(cache_key_3))
self.assertEqual(cache.get(cache_key_main), '4')
self.assertEqual(stored_data, pickled_data_1 + pickled_data_2)
# Try fetching the data we stored.
cache_func.spy.reset_calls()
data_yielded = []
result = cache_memoize_iter(cache_key,
cache_func,
compress_large_data=False)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(next(result), data1)
self.assertEqual(next(result), data2)
with self.assertRaises(StopIteration):
next(result)
self.assertEqual(data_yielded, [])
self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_uncompressed(self):
"""Testing cache_memoize_iter without compression"""
cache_key = 'abc123'
data_yielded = []
data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2)
data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2)
def cache_func():
data_yielded.append('data1')
yield data1
data_yielded.append('data2')
yield data2
self.spy_on(cache_func, call_original=True)
result = cache_memoize_iter(cache_key, cache_func,
compress_large_data=False)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(data_yielded, [])
self.assertEqual(next(result), data1)
self.assertEqual(data_yielded, ['data1'])
self.assertEqual(next(result), data2)
self.assertEqual(data_yielded, ['data1', 'data2'])
with self.assertRaises(StopIteration):
next(result)
self.assertTrue(cache_func.spy.called)
cache_key_main = make_cache_key(cache_key)
cache_key_0 = make_cache_key('%s-0' % cache_key)
cache_key_1 = make_cache_key('%s-1' % cache_key)
cache_key_2 = make_cache_key('%s-2' % cache_key)
cache_key_3 = make_cache_key('%s-3' % cache_key)
self.assertTrue(cache_key_main in cache)
self.assertTrue(cache_key_0 in cache)
self.assertTrue(cache_key_1 in cache)
self.assertTrue(cache_key_2 in cache)
self.assertTrue(cache_key_3 in cache)
self.assertFalse(make_cache_key('%s-4' % cache_key) in cache)
# Verify the contents of the stored data.
stored_data = b''.join(cache.get(cache_key_0) +
cache.get(cache_key_1) +
cache.get(cache_key_2) +
cache.get(cache_key_3))
self.assertEqual(cache.get(cache_key_main), '4')
self.assertEqual(stored_data, pickled_data_1 + pickled_data_2)
# Try fetching the data we stored.
cache_func.spy.reset_calls()
data_yielded = []
result = cache_memoize_iter(cache_key, cache_func,
compress_large_data=False)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(next(result), data1)
self.assertEqual(next(result), data2)
with self.assertRaises(StopIteration):
next(result)
self.assertEqual(data_yielded, [])
self.assertFalse(cache_func.spy.called)
