def test_cache_memoize_iter_compressed(self): """Testing cache_memoize_iter with compression""" cache_key = 'abc123' data_yielded = [] # This takes into account the size of the pickle data, and will # get us to exactly 2 chunks of data in cache, each. data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2) data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2) def cache_func(): data_yielded.append('data1') yield data1 data_yielded.append('data2') yield data2 self.spy_on(cache_func, call_original=True) result = cache_memoize_iter(cache_key, cache_func, compress_large_data=True) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(data_yielded, []) self.assertEqual(next(result), data1) self.assertEqual(data_yielded, ['data1']) self.assertEqual(next(result), data2) self.assertEqual(data_yielded, ['data1', 'data2']) with self.assertRaises(StopIteration): next(result) self.assertTrue(cache_func.spy.called) cache_key_main = make_cache_key(cache_key) cache_key_0 = make_cache_key('%s-0' % cache_key) self.assertTrue(cache_key_main in cache) self.assertTrue(cache_key_0 in cache) self.assertFalse(make_cache_key('%s-1' % cache_key) in cache) # Verify the contents of the stored data. self.assertEqual(cache.get(cache_key_main), '1') self.assertEqual( cache.get(cache_key_0)[0], zlib.compress(pickled_data_1 + pickled_data_2)) # Try fetching the data we stored. cache_func.spy.reset_calls() data_yielded = [] result = cache_memoize_iter(cache_key, cache_func, compress_large_data=True) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(next(result), data1) self.assertEqual(next(result), data2) with self.assertRaises(StopIteration): next(result) self.assertEqual(data_yielded, []) self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_compressed(self): """Testing cache_memoize_iter with compression""" cache_key = 'abc123' data_yielded = [] # This takes into account the size of the pickle data, and will # get us to exactly 2 chunks of data in cache, each. data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2) data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2) def cache_func(): data_yielded.append('data1') yield data1 data_yielded.append('data2') yield data2 self.spy_on(cache_func, call_original=True) result = cache_memoize_iter(cache_key, cache_func, compress_large_data=True) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(data_yielded, []) self.assertEqual(next(result), data1) self.assertEqual(data_yielded, ['data1']) self.assertEqual(next(result), data2) self.assertEqual(data_yielded, ['data1', 'data2']) with self.assertRaises(StopIteration): next(result) self.assertTrue(cache_func.spy.called) cache_key_main = make_cache_key(cache_key) cache_key_0 = make_cache_key('%s-0' % cache_key) self.assertTrue(cache_key_main in cache) self.assertTrue(cache_key_0 in cache) self.assertFalse(make_cache_key('%s-1' % cache_key) in cache) # Verify the contents of the stored data. self.assertEqual(cache.get(cache_key_main), '1') self.assertEqual(cache.get(cache_key_0)[0], zlib.compress(pickled_data_1 + pickled_data_2)) # Try fetching the data we stored. cache_func.spy.reset_calls() data_yielded = [] result = cache_memoize_iter(cache_key, cache_func, compress_large_data=True) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(next(result), data1) self.assertEqual(next(result), data2) with self.assertRaises(StopIteration): next(result) self.assertEqual(data_yielded, []) self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_uncompressed(self): """Testing cache_memoize_iter without compression""" cache_key = 'abc123' data_yielded = [] data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2) data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2) def cache_func(): data_yielded.append('data1') yield data1 data_yielded.append('data2') yield data2 self.spy_on(cache_func, call_original=True) result = cache_memoize_iter(cache_key, cache_func, compress_large_data=False) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(data_yielded, []) self.assertEqual(next(result), data1) self.assertEqual(data_yielded, ['data1']) self.assertEqual(next(result), data2) self.assertEqual(data_yielded, ['data1', 'data2']) with self.assertRaises(StopIteration): next(result) self.assertTrue(cache_func.spy.called) cache_key_main = make_cache_key(cache_key) cache_key_0 = make_cache_key('%s-0' % cache_key) cache_key_1 = make_cache_key('%s-1' % cache_key) cache_key_2 = make_cache_key('%s-2' % cache_key) cache_key_3 = make_cache_key('%s-3' % cache_key) self.assertTrue(cache_key_main in cache) self.assertTrue(cache_key_0 in cache) self.assertTrue(cache_key_1 in cache) self.assertTrue(cache_key_2 in cache) self.assertTrue(cache_key_3 in cache) self.assertFalse(make_cache_key('%s-4' % cache_key) in cache) # Verify the contents of the stored data. stored_data = b''.join( cache.get(cache_key_0) + cache.get(cache_key_1) + cache.get(cache_key_2) + cache.get(cache_key_3)) self.assertEqual(cache.get(cache_key_main), '4') self.assertEqual(stored_data, pickled_data_1 + pickled_data_2) # Try fetching the data we stored. cache_func.spy.reset_calls() data_yielded = [] result = cache_memoize_iter(cache_key, cache_func, compress_large_data=False) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(next(result), data1) self.assertEqual(next(result), data2) with self.assertRaises(StopIteration): next(result) self.assertEqual(data_yielded, []) self.assertFalse(cache_func.spy.called)
def test_cache_memoize_iter_uncompressed(self): """Testing cache_memoize_iter without compression""" cache_key = 'abc123' data_yielded = [] data1, pickled_data_1 = self._build_test_chunk_data(num_chunks=2) data2, pickled_data_2 = self._build_test_chunk_data(num_chunks=2) def cache_func(): data_yielded.append('data1') yield data1 data_yielded.append('data2') yield data2 self.spy_on(cache_func, call_original=True) result = cache_memoize_iter(cache_key, cache_func, compress_large_data=False) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(data_yielded, []) self.assertEqual(next(result), data1) self.assertEqual(data_yielded, ['data1']) self.assertEqual(next(result), data2) self.assertEqual(data_yielded, ['data1', 'data2']) with self.assertRaises(StopIteration): next(result) self.assertTrue(cache_func.spy.called) cache_key_main = make_cache_key(cache_key) cache_key_0 = make_cache_key('%s-0' % cache_key) cache_key_1 = make_cache_key('%s-1' % cache_key) cache_key_2 = make_cache_key('%s-2' % cache_key) cache_key_3 = make_cache_key('%s-3' % cache_key) self.assertTrue(cache_key_main in cache) self.assertTrue(cache_key_0 in cache) self.assertTrue(cache_key_1 in cache) self.assertTrue(cache_key_2 in cache) self.assertTrue(cache_key_3 in cache) self.assertFalse(make_cache_key('%s-4' % cache_key) in cache) # Verify the contents of the stored data. stored_data = b''.join(cache.get(cache_key_0) + cache.get(cache_key_1) + cache.get(cache_key_2) + cache.get(cache_key_3)) self.assertEqual(cache.get(cache_key_main), '4') self.assertEqual(stored_data, pickled_data_1 + pickled_data_2) # Try fetching the data we stored. cache_func.spy.reset_calls() data_yielded = [] result = cache_memoize_iter(cache_key, cache_func, compress_large_data=False) self.assertTrue(inspect.isgenerator(result)) self.assertEqual(next(result), data1) self.assertEqual(next(result), data2) with self.assertRaises(StopIteration): next(result) self.assertEqual(data_yielded, []) self.assertFalse(cache_func.spy.called)