def update_objects_to_be_complete(youtube_ids_in_filesystem): # Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False updated_video_ids = [] for chunk in break_into_chunks(youtube_ids_in_filesystem): video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk) video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False) updated_video_ids += [i18n.get_video_id(video_file.youtube_id) for video_file in video_files_needing_model_update] if updated_video_ids: caching.invalidate_all_caches() self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % len(updated_video_ids)) return updated_video_ids
def add_missing_objects_to_db(youtube_ids_in_filesystem, videos_marked_at_all): # Files that exist, but are not in the DB, should be assumed to be good videos, # and just needing to be added to the DB. Add them to the DB in this way, # so that these files also trigger the update code below (and trigger cache invalidation) youtube_ids_needing_model_creation = list(youtube_ids_in_filesystem - videos_marked_at_all) new_video_files = [] if youtube_ids_needing_model_creation: for lang_code, youtube_ids in divide_videos_by_language(youtube_ids_needing_model_creation).iteritems(): # OK to do bulk_create; cache invalidation triggered via save download lang_video_files = [VideoFile(youtube_id=id, percent_complete=100, download_in_progress=False, language=lang_code) for id in youtube_ids] VideoFile.objects.bulk_create(lang_video_files) new_video_files += lang_video_files caching.invalidate_all_caches() # Do this within the loop, to update users ASAP self.stdout.write("Created %d VideoFile models (and marked them as complete, since the files exist)\n" % len(new_video_files)) return [i18n.get_video_id(video_file.youtube_id) for video_file in new_video_files]
def test_cache_invalidation(self): """Create the cache item, then invalidate it and show that it is deleted.""" # Get a random video id n_videos = len(self.video_cache) video_id = self.video_cache.keys()[10]#random.choice(self.video_cache.keys()) sys.stdout.write("Testing on video_id = %s\n" % video_id) video_path = self.video_cache[video_id][0]['path'] # Clean the cache for this item caching.expire_page(path=video_path, failure_ok=True) # Create the cache item, and check it self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page") caching.regenerate_all_pages_related_to_videos(video_ids=[video_id]) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") # Invalidate the cache item, and check it caching.invalidate_all_caches() # test the convenience function self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def handle(self, *args, **options): if settings.CENTRAL_SERVER: raise CommandError("This must only be run on distributed servers server.") lang_code = lcode_to_ietf(options["lang_code"]) software_version = options["software_version"] logging.info("Downloading language pack for lang_code=%s, software_version=%s" % (lang_code, software_version)) # Download the language pack try: if options['file']: self.start(_("Using local language pack '%(filepath)s'") % {"filepath": options['file']}) zip_filepath = options['file'] else: self.start(_("Downloading language pack '%(lang_code)s'") % {"lang_code": lang_code}) zip_filepath = get_language_pack(lang_code, software_version, callback=self.cb) # Unpack into locale directory self.next_stage(_("Unpacking language pack '%(lang_code)s'") % {"lang_code": lang_code}) unpack_language(lang_code, zip_filepath=zip_filepath) # self.next_stage(_("Creating static files for language pack '%(lang_code)s'") % {"lang_code": lang_code}) update_jsi18n_file(lang_code) self.next_stage(_("Moving files to their appropriate local disk locations.")) move_dubbed_video_map(lang_code) move_exercises(lang_code) move_srts(lang_code) move_video_sizes_file(lang_code) self.next_stage(_("Invalidate caches")) caching.invalidate_all_caches() self.complete(_("Finished processing language pack %(lang_code)s") % {"lang_code": lang_code}) except Exception as e: self.cancel(stage_status="error", notes=_("Error: %(error_msg)s") % {"error_msg": unicode(e)}) raise