def test_cache_invalidation(self): """Create the cache item, then invalidate it and show that it is deleted.""" # Get a random youtube id n_videos = len(topicdata.NODE_CACHE['Video']) video_slug = topicdata.NODE_CACHE['Video'].keys()[ 10] #random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_slug = %s\n" % video_slug) youtube_id = topicdata.NODE_CACHE['Video'][video_slug]['youtube_id'] video_path = topicdata.NODE_CACHE['Video'][video_slug]['paths'][0] # Clean the cache for this item caching.expire_page(path=video_path) # Create the cache item, and check it self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page") caching.regenerate_all_pages_related_to_videos(video_ids=[youtube_id]) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") # Invalidate the cache item, and check it caching.invalidate_all_pages_related_to_video( video_id=youtube_id) # test the convenience function self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def test_cache_across_clients(self): """Show that caching is accessible across all clients (i.e. that different clients don't generate different cache keys)""" # Get a random video id n_videos = len(topicdata.NODE_CACHE['Video']) video_id = random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_id = %s\n" % video_id) video_path = topicdata.NODE_CACHE['Video'][video_id][0]['path'] # Clean the cache for this item caching.expire_page(path=video_path, failure_ok=True) self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Set up the cache with Django client Client().get(video_path) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Get the same cache key when getting with urllib, and make sure the cache is created again urllib.urlopen(self.live_server_url + video_path).close() self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after urllib get") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Same deal, now using requests library requests.get(self.live_server_url + video_path) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after requestsget") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page")
def show_cache(self, force=False): """Go through each cacheable page, and show which are cached and which are NOT""" for node_type in ['Topic', 'Video', 'Exercise']: self.stdout.write("Cached %ss:\n" % node_type) for narr in topic_tools.get_node_cache(node_type).values(): for n in narr: if caching.has_cache_key(path=n["path"]): self.stdout.write("\t%s\n" % n["path"])
def show_cache(self, force=False): """Go through each cacheable page, and show which are cached and which are NOT""" for node_type in ['Topic', 'Video', 'Exercise']: self.stdout.write("Cached %ss:\n" % node_type) for narr in topicdata.NODE_CACHE[node_type].values(): for n in narr: if caching.has_cache_key(path=n["path"]): self.stdout.write("\t%s\n" % n["path"])
def show_cache(self, force=False): """Go through each cacheable page, and show which are cached and which are NOT""" for node_type in ['Topic', 'Video', 'Exercise']: self.stdout.write("Cached %ss:\n" % node_type) for n in topicdata.NODE_CACHE[node_type].values(): for path in n["paths"] if node_type in topic_tools.multipath_kinds else [n["path"]]: if caching.has_cache_key(path=path): self.stdout.write("\t%s\n" % path)
def clear_cache(self): """Go through each cacheable page, and show which are cached and which are NOT""" for node_type in ['Topic', 'Video', 'Exercise']: self.stdout.write("Clearing %ss:\n" % node_type) for narr in topicdata.NODE_CACHE[node_type].values(): for n in narr: if caching.has_cache_key(path=n["path"]): self.stdout.write("\t%s\n" % n["path"]) caching.expire_page(path=n["path"])
def test_video_deleted_with_cache(self): """ Run videoscan to create cache items, then re-run to verify that the cache is cleared. """ out = call_command("videoscan", auto_cache=True) cached_paths = caching.get_video_page_paths(video_id=self.video_id) for path in cached_paths: self.assertTrue(caching.has_cache_key(path), "Check that cache has path %s" % path) self.assertTrue(os.path.exists(self.fake_video_file), "Check that video file exists.") # Remove the video os.remove(self.fake_video_file) self.assertFalse(os.path.exists(self.fake_video_file), "Check that video file no longer exists.") # Call videoscan, and validate. out = call_command("videoscan") self.assertEqual(VideoFile.objects.all().count(), 0, "Make sure there are now no VideoFile objects.") for path in cached_paths: self.assertFalse(caching.has_cache_key(path), "Check that cache does NOT have path %s" % path)
def test_cache_invalidation(self): """Create the cache item, then invalidate it and show that it is deleted.""" # Get a random video id n_videos = len(topicdata.NODE_CACHE['Video']) video_id = topicdata.NODE_CACHE['Video'].keys()[10]#random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_id = %s\n" % video_id) video_path = topicdata.NODE_CACHE['Video'][video_id][0]['path'] # Clean the cache for this item caching.expire_page(path=video_path, failure_ok=True) # Create the cache item, and check it self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page") caching.regenerate_all_pages_related_to_videos(video_ids=[video_id]) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") # Invalidate the cache item, and check it caching.invalidate_all_pages_related_to_video(video_id=video_id) # test the convenience function self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def show_cache(self, force=False): """Go through each cacheable page, and show which are cached and which are NOT""" for node_type in ['Topic', 'Video', 'Exercise']: self.stdout.write("Cached %ss:\n" % node_type) for n in topicdata.NODE_CACHE[node_type].values(): for path in n[ "paths"] if node_type in topic_tools.multipath_kinds else [ n["path"] ]: if caching.has_cache_key(path=path): self.stdout.write("\t%s\n" % path)
def test_video_added_with_cache(self): """ Add a video in the filesystem, call videoscan to create the videofile object and cache items """ # Call videoscan, and validate. out = call_command("videoscan", auto_cache=True) self.assertEqual(VideoFile.objects.all().count(), 1, "Make sure there is now one VideoFile object.") self.assertEqual(VideoFile.objects.all()[0].youtube_id, self.video_id, "Make sure the video is the one we created.") self.assertTrue(self.web_cache._num_entries > 0, "Check that cache is not empty.") cached_paths = caching.get_video_page_paths(video_id=self.video_id) for path in cached_paths: self.assertTrue(caching.has_cache_key(path), "Check that cache has path %s" % path)
def create_page_cache(self, path, force=False): """Go through each cacheable page, and either: (a) Cache each page that is not or (b) Kill and regenerate each page""" if force: if caching.has_cache_key(path=path): caching.expire_page(path=path) self.stdout.write("[Redo]\t%s\n" % path) else: self.stdout.write("[Miss]\t%s\n" % path) caching.create_cache(path=path) else: if not caching.has_cache_key(path=path): caching.create_cache(path=path) self.stdout.write("[Miss]\t%s\n" % path) if not caching.has_cache_key(path=path): # should never get here! self.stdout.write("%s%s\n" % ("?"*10, path))
def create_page_cache(self, path, force=False): """Go through each cacheable page, and either: (a) Cache each page that is not or (b) Kill and regenerate each page""" if force: if caching.has_cache_key(path=path): caching.expire_page(path=path) self.stdout.write("[Redo]\t%s\n" % path) else: self.stdout.write("[Miss]\t%s\n" % path) caching.create_cache(path=path) else: if not caching.has_cache_key(path=path): caching.create_cache(path=path) self.stdout.write("[Miss]\t%s\n" % path) if not caching.has_cache_key(path=path): # should never get here! self.stdout.write("%s%s\n" % ("?" * 10, path))
def test_cache_across_clients(self): """Show that caching is accessible across all clients (i.e. that different clients don't generate different cache keys)""" # Get a random youtube id n_videos = len(topicdata.NODE_CACHE['Video']) video_slug = random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_slug = %s\n" % video_slug) youtube_id = topicdata.NODE_CACHE['Video'][video_slug]['youtube_id'] video_path = topicdata.NODE_CACHE['Video'][video_slug]['paths'][0] # Clean the cache for this item caching.expire_page(path=video_path) self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Set up the cache with Django client Client().get(video_path) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Get the same cache key when getting with urllib, and make sure the cache is created again urllib.urlopen(self.live_server_url + video_path).close() self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after urllib get") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page") # Same deal, now using requests library requests.get(self.live_server_url + video_path) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after requestsget") caching.expire_page(path=video_path) # clean cache self.assertTrue(not caching.has_cache_key(path=video_path), "expect: No cache key after expiring the page")