def test_cache_invalidation(self): """Create the cache item, then invalidate it and show that it is deleted.""" # Get a random youtube id n_videos = len(topicdata.NODE_CACHE['Video']) video_slug = topicdata.NODE_CACHE['Video'].keys()[ 10] #random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_slug = %s\n" % video_slug) youtube_id = topicdata.NODE_CACHE['Video'][video_slug]['youtube_id'] video_path = topicdata.NODE_CACHE['Video'][video_slug]['paths'][0] # Clean the cache for this item caching.expire_page(path=video_path) # Create the cache item, and check it self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page") caching.regenerate_all_pages_related_to_videos(video_ids=[youtube_id]) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") # Invalidate the cache item, and check it caching.invalidate_all_pages_related_to_video( video_id=youtube_id) # test the convenience function self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def handle(self, *args, **options): caching_enabled = (settings.CACHE_TIME != 0) touched_video_ids = [] # delete VideoFile objects that are not marked as in progress, but are neither 0% nor 100% done; they're broken video_files_to_delete = VideoFile.objects.filter(download_in_progress=False, percent_complete__gt=0, percent_complete__lt=100) youtube_ids_to_delete = [d["youtube_id"] for d in video_files_to_delete.values("youtube_id")] video_files_to_delete.delete() touched_video_ids += [i18n.get_video_id(yid) for yid in youtube_ids_to_delete] if len(video_files_to_delete): self.stdout.write("Deleted %d VideoFile models (to mark them as not downloaded, since they were in a bad state)\n" % len(video_files_to_delete)) files = glob.glob(settings.CONTENT_ROOT + "*.mp4") videos_marked_at_all = set([video.youtube_id for video in VideoFile.objects.all()]) videos_marked_as_in_progress = set([video.youtube_id for video in VideoFile.objects.filter(download_in_progress=True)]) videos_marked_as_unstarted = set([video.youtube_id for video in VideoFile.objects.filter(percent_complete=0, download_in_progress=False)]) videos_flagged_for_download = set([video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True)]) videos_in_filesystem = set([path.replace("\\", "/").split("/")[-1].split(".")[0] for path in files]) # Files that exist, but are not in the DB, should be assumed to be good videos, # and just needing to be added to the DB. Add them to the DB in this way, # so that these files also trigger the update code below (and trigger cache invalidation) video_ids_needing_model_creation = list(videos_in_filesystem - videos_marked_at_all) count = len(video_ids_needing_model_creation) if count: # OK to do bulk_create; cache invalidation triggered via save download VideoFile.objects.bulk_create([VideoFile(youtube_id=id, percent_complete=0, download_in_progress=False) for id in video_ids_needing_model_creation]) self.stdout.write("Created %d VideoFile models (to mark them as complete, since the files exist)\n" % len(video_ids_needing_model_creation)) touched_video_ids += [i18n.get_video_id(yid) or yid for yid in video_ids_needing_model_creation] # Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False # These should be individually saved to be 100% complete, to trigger their availability (and cache invalidation) count = 0 for chunk in break_into_chunks(videos_in_filesystem): video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() for videofile in video_files_needing_model_update: videofile.percent_complete = 100 videofile.flagged_for_download = False videofile.save() if count: self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % count) # VideoFile objects say they're available, but that don't actually exist. count = 0 videos_needing_model_deletion_chunked = break_into_chunks(videos_marked_at_all - videos_in_filesystem - videos_flagged_for_download) for chunk in videos_needing_model_deletion_chunked: video_files_needing_model_deletion = VideoFile.objects.filter(youtube_id__in=chunk) count += video_files_needing_model_deletion.count() video_files_needing_model_deletion.delete() touched_video_ids += [i18n.get_video_id(yid) or yid for yid in chunk] if count: self.stdout.write("Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % count) if options["auto_cache"] and caching_enabled and touched_video_ids: caching.regenerate_all_pages_related_to_videos(video_ids=list(set(touched_video_ids)))
def handle(self, *args, **options): caching_enabled = settings.CACHE_TIME != 0 handled_video_ids = [] while True: # loop until the method is aborted if VideoFile.objects.filter(download_in_progress=True).count() > 0: self.stderr.write("Another download is still in progress; aborting.\n") break videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") break video = videos[0] if video.cancel_download == True: video.download_in_progress = False video.save() self.stdout.write("Download cancelled; aborting.\n") break video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) try: download_video(video.youtube_id, callback=download_progress_callback(self, video)) self.stdout.write("Download is complete!\n") except Exception as e: self.stderr.write("Error in downloading: %s\n" % e) video.download_in_progress = False video.save() force_job("videodownload", "Download Videos") # infinite recursive call? :( break handled_video_ids.append(video.youtube_id) # Expire, but don't regenerate until the very end, for efficiency. if caching_enabled: caching.invalidate_all_pages_related_to_video(video_id=video.youtube_id) if options["auto_cache"] and caching_enabled and handled_video_ids: caching.regenerate_all_pages_related_to_videos(video_ids=handled_video_ids)
def test_cache_invalidation(self): """Create the cache item, then invalidate it and show that it is deleted.""" # Get a random video id n_videos = len(topicdata.NODE_CACHE['Video']) video_id = topicdata.NODE_CACHE['Video'].keys()[10]#random.choice(topicdata.NODE_CACHE['Video'].keys()) sys.stdout.write("Testing on video_id = %s\n" % video_id) video_path = topicdata.NODE_CACHE['Video'][video_id][0]['path'] # Clean the cache for this item caching.expire_page(path=video_path, failure_ok=True) # Create the cache item, and check it self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page") caching.regenerate_all_pages_related_to_videos(video_ids=[video_id]) self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get") # Invalidate the cache item, and check it caching.invalidate_all_pages_related_to_video(video_id=video_id) # test the convenience function self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def handle(self, *args, **options): caching_enabled = settings.CACHE_TIME != 0 handled_video_ids = [] # stored to deal with caching while True: # loop until the method is aborted if VideoFile.objects.filter(download_in_progress=True).count() > 0: self.stderr.write( "Another download is still in progress; aborting.\n") break # Grab any video that hasn't been tried yet videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") break video = videos[0] # User intervention if video.cancel_download == True: video.download_in_progress = False video.save() self.stdout.write("Download cancelled; aborting.\n") break # Grab a video as OURS to handle, set fields to indicate to others that we're on it! video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) try: download_video(video.youtube_id, callback=download_progress_callback( self, video)) handled_video_ids.append(video.youtube_id) self.stdout.write("Download is complete!\n") except Exception as e: if isinstance(e, URLNotFound): # This should never happen, but if it does, remove the VideoFile from the queue, and continue # to the next video. Warning: this will leave the update page in a weird state, currently # (and require a refresh of the update page in order to start showing progress again) video.delete() continue # On connection error, report the error, mark the video as not downloaded, and give up for now. self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e)) video.download_in_progress = False video.percent_complete = 0 video.save() break # Expire, but don't regenerate until the very end, for efficiency. if caching_enabled: caching.invalidate_all_pages_related_to_video( video_id=video.youtube_id) # After all is done, regenerate all pages # since this is computationally intensive, only do it after we're sure # nothing more will change (so that we don't regenerate something that is # later invalidated by another video downloaded in the loop) if options["auto_cache"] and caching_enabled and handled_video_ids: caching.regenerate_all_pages_related_to_videos( video_ids=handled_video_ids)
def handle(self, *args, **options): caching_enabled = (settings.CACHE_TIME != 0) touched_video_ids = [] # delete VideoFile objects that are not marked as in progress, but are neither 0% nor 100% done; they're broken video_files_to_delete = VideoFile.objects.filter( download_in_progress=False, percent_complete__gt=0, percent_complete__lt=100) youtube_ids_to_delete = [ d["youtube_id"] for d in video_files_to_delete.values("youtube_id") ] video_files_to_delete.delete() if caching_enabled: for youtube_id in youtube_ids_to_delete: caching.invalidate_all_pages_related_to_video( video_id=youtube_id) touched_video_ids.append(youtube_id) if len(video_files_to_delete): self.stdout.write( "Deleted %d VideoFile models (to mark them as not downloaded, since they were in a bad state)\n" % len(video_files_to_delete)) files = glob.glob(settings.CONTENT_ROOT + "*.mp4") subtitle_files = glob.glob(settings.CONTENT_ROOT + "*.srt") videos_marked_at_all = set( [video.youtube_id for video in VideoFile.objects.all()]) videos_marked_as_in_progress = set([ video.youtube_id for video in VideoFile.objects.filter(download_in_progress=True) ]) videos_marked_as_unstarted = set([ video.youtube_id for video in VideoFile.objects.filter(percent_complete=0, download_in_progress=False) ]) videos_in_filesystem = set([ path.replace("\\", "/").split("/")[-1].split(".")[0] for path in files ]) videos_in_filesystem_chunked = break_into_chunks(videos_in_filesystem) videos_flagged_for_download = set([ video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True) ]) subtitles_in_filesystem = set([ path.replace("\\", "/").split("/")[-1].split(".")[0] for path in subtitle_files ]) subtitles_in_filesystem_chunked = break_into_chunks( subtitles_in_filesystem) count = 0 for chunk in videos_in_filesystem_chunked: video_files_needing_model_update = VideoFile.objects.filter( percent_complete=0, download_in_progress=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False) if caching_enabled: for vf in video_files_needing_model_update: caching.invalidate_all_pages_related_to_video( video_id=vf.youtube_id) touched_video_ids.append(vf.youtube_id) if count: self.stdout.write( "Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % count) video_ids_needing_model_creation = list(videos_in_filesystem - videos_marked_at_all) count = len(video_ids_needing_model_creation) if count: VideoFile.objects.bulk_create([ VideoFile(youtube_id=youtube_id, percent_complete=100) for youtube_id in video_ids_needing_model_creation ]) if caching_enabled: for vid in video_ids_needing_model_creation: caching.invalidate_all_pages_related_to_video(video_id=vid) touched_video_ids.append(vid) self.stdout.write( "Created %d VideoFile models (to mark them as complete, since the files exist)\n" % count) count = 0 videos_needing_model_deletion_chunked = break_into_chunks( videos_marked_at_all - videos_in_filesystem - videos_flagged_for_download) for chunk in videos_needing_model_deletion_chunked: video_files_needing_model_deletion = VideoFile.objects.filter( youtube_id__in=chunk) count += video_files_needing_model_deletion.count() video_files_needing_model_deletion.delete() if caching_enabled: for video_id in chunk: caching.invalidate_all_pages_related_to_video( video_id=video_id) touched_video_ids.append(video_id) if count: self.stdout.write( "Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % count) count = 0 for chunk in subtitles_in_filesystem_chunked: video_files_needing_model_update = VideoFile.objects.filter( subtitle_download_in_progress=False, subtitles_downloaded=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() video_files_needing_model_update.update(subtitles_downloaded=True) if caching_enabled: for vf in video_files_needing_model_update: caching.invalidate_all_pages_related_to_video( video_id=vf.youtube_id) touched_video_ids.append(vf.youtube_id) if count: self.stdout.write( "Updated %d VideoFile models (marked them as having subtitles)\n" % count) if options["auto_cache"] and caching_enabled and touched_video_ids: caching.regenerate_all_pages_related_to_videos( video_ids=touched_video_ids)
def handle(self, *args, **options): caching_enabled = settings.CACHE_TIME != 0 handled_video_ids = [] # stored to deal with caching while True: # loop until the method is aborted if VideoFile.objects.filter(download_in_progress=True).count() > 0: self.stderr.write("Another download is still in progress; aborting.\n") break # Grab any video that hasn't been tried yet videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") break video = videos[0] # User intervention if video.cancel_download == True: video.download_in_progress = False video.save() self.stdout.write("Download cancelled; aborting.\n") break # Grab a video as OURS to handle, set fields to indicate to others that we're on it! video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) try: download_video(video.youtube_id, callback=download_progress_callback(self, video)) handled_video_ids.append(video.youtube_id) self.stdout.write("Download is complete!\n") except Exception as e: if isinstance(e, URLNotFound): # This should never happen, but if it does, remove the VideoFile from the queue, and continue # to the next video. Warning: this will leave the update page in a weird state, currently # (and require a refresh of the update page in order to start showing progress again) video.delete() continue # On connection error, report the error, mark the video as not downloaded, and give up for now. self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e)) video.download_in_progress = False video.percent_complete = 0 video.save() break # Expire, but don't regenerate until the very end, for efficiency. if caching_enabled: caching.invalidate_all_pages_related_to_video(video_id=video.youtube_id) # After all is done, regenerate all pages # since this is computationally intensive, only do it after we're sure # nothing more will change (so that we don't regenerate something that is # later invalidated by another video downloaded in the loop) if options["auto_cache"] and caching_enabled and handled_video_ids: caching.regenerate_all_pages_related_to_videos(video_ids=handled_video_ids)
def handle(self, *args, **options): self.video = None handled_youtube_ids = [] # stored to deal with caching failed_youtube_ids = [] # stored to avoid requerying failures. set_process_priority.lowest(logging=settings.LOG) try: while True: # loop until the method is aborted # Grab any video that hasn't been tried yet videos = VideoFile.objects \ .filter(flagged_for_download=True, download_in_progress=False) \ .exclude(youtube_id__in=failed_youtube_ids) video_count = videos.count() if video_count == 0: self.stdout.write("Nothing to download; exiting.\n") break # Grab a video as OURS to handle, set fields to indicate to others that we're on it! # Update the video logging video = videos[0] video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) # Update the progress logging self.set_stages( num_stages=video_count + len(handled_youtube_ids) + len(failed_youtube_ids) + int(options["auto_cache"])) if not self.started(): self.start(stage_name=video.youtube_id) # Initiate the download process try: download_video(video.youtube_id, callback=partial( self.download_progress_callback, video)) handled_youtube_ids.append(video.youtube_id) self.stdout.write("Download is complete!\n") except Exception as e: # On error, report the error, mark the video as not downloaded, # and allow the loop to try other videos. self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e)) video.download_in_progress = False video.flagged_for_download = not isinstance( e, URLNotFound) # URLNotFound means, we won't try again video.save() # Rather than getting stuck on one video, continue to the next video. failed_youtube_ids.append(video.youtube_id) continue # This can take a long time, without any further update, so ... best to avoid. if options["auto_cache"] and caching.caching_is_enabled( ) and handled_youtube_ids: self.update_stage( stage_name=self.video.youtube_id, stage_percent=0, notes=_("Generating all pages related to videos.")) caching.regenerate_all_pages_related_to_videos(video_ids=list( set([ i18n.get_video_id(yid) or yid for yid in handled_youtube_ids ]))) # Update self.complete( notes= _("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully." ) % { "num_handled_videos": len(handled_youtube_ids), "num_total_videos": len(handled_youtube_ids) + len(failed_youtube_ids), }) except Exception as e: sys.stderr.write("Error: %s\n" % e) self.cancel(notes=_("Error: %s") % e)
def handle(self, *args, **options): caching_enabled = (settings.CACHE_TIME != 0) touched_video_ids = [] # delete VideoFile objects that are not marked as in progress, but are neither 0% nor 100% done; they're broken video_files_to_delete = VideoFile.objects.filter(download_in_progress=False, percent_complete__gt=0, percent_complete__lt=100) youtube_ids_to_delete = [d["youtube_id"] for d in video_files_to_delete.values("youtube_id")] video_files_to_delete.delete() if caching_enabled: for youtube_id in youtube_ids_to_delete: caching.invalidate_all_pages_related_to_video(video_id=youtube_id) touched_video_ids.append(youtube_id) if len(video_files_to_delete): self.stdout.write("Deleted %d VideoFile models (to mark them as not downloaded, since they were in a bad state)\n" % len(video_files_to_delete)) files = glob.glob(settings.CONTENT_ROOT + "*.mp4") subtitle_files = glob.glob(settings.CONTENT_ROOT + "*.srt") videos_marked_at_all = set([video.youtube_id for video in VideoFile.objects.all()]) videos_marked_as_in_progress = set([video.youtube_id for video in VideoFile.objects.filter(download_in_progress=True)]) videos_marked_as_unstarted = set([video.youtube_id for video in VideoFile.objects.filter(percent_complete=0, download_in_progress=False)]) videos_in_filesystem = set([path.replace("\\", "/").split("/")[-1].split(".")[0] for path in files]) videos_in_filesystem_chunked = break_into_chunks(videos_in_filesystem) videos_flagged_for_download = set([video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True)]) subtitles_in_filesystem = set([path.replace("\\", "/").split("/")[-1].split(".")[0] for path in subtitle_files]) subtitles_in_filesystem_chunked = break_into_chunks(subtitles_in_filesystem) count = 0 for chunk in videos_in_filesystem_chunked: video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False) if caching_enabled: for vf in video_files_needing_model_update: caching.invalidate_all_pages_related_to_video(video_id=vf.youtube_id) touched_video_ids.append(vf.youtube_id) if count: self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % count) video_ids_needing_model_creation = list(videos_in_filesystem - videos_marked_at_all) count = len(video_ids_needing_model_creation) if count: VideoFile.objects.bulk_create([VideoFile(youtube_id=youtube_id, percent_complete=100) for youtube_id in video_ids_needing_model_creation]) if caching_enabled: for vid in video_ids_needing_model_creation: caching.invalidate_all_pages_related_to_video(video_id=vid) touched_video_ids.append(vid) self.stdout.write("Created %d VideoFile models (to mark them as complete, since the files exist)\n" % count) count = 0 videos_needing_model_deletion_chunked = break_into_chunks(videos_marked_at_all - videos_in_filesystem - videos_flagged_for_download) for chunk in videos_needing_model_deletion_chunked: video_files_needing_model_deletion = VideoFile.objects.filter(youtube_id__in=chunk) count += video_files_needing_model_deletion.count() video_files_needing_model_deletion.delete() if caching_enabled: for video_id in chunk: caching.invalidate_all_pages_related_to_video(video_id=video_id) touched_video_ids.append(video_id) if count: self.stdout.write("Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % count) count = 0 for chunk in subtitles_in_filesystem_chunked: video_files_needing_model_update = VideoFile.objects.filter(subtitle_download_in_progress=False, subtitles_downloaded=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() video_files_needing_model_update.update(subtitles_downloaded=True) if caching_enabled: for vf in video_files_needing_model_update: caching.invalidate_all_pages_related_to_video(video_id=vf.youtube_id) touched_video_ids.append(vf.youtube_id) if count: self.stdout.write("Updated %d VideoFile models (marked them as having subtitles)\n" % count) if options["auto_cache"] and caching_enabled and touched_video_ids: caching.regenerate_all_pages_related_to_videos(video_ids=touched_video_ids)
def handle(self, *args, **options): caching_enabled = (settings.CACHE_TIME != 0) touched_video_ids = [] # delete VideoFile objects that are not marked as in progress, but are neither 0% nor 100% done; they're broken video_files_to_delete = VideoFile.objects.filter( download_in_progress=False, percent_complete__gt=0, percent_complete__lt=100) youtube_ids_to_delete = [ d["youtube_id"] for d in video_files_to_delete.values("youtube_id") ] video_files_to_delete.delete() touched_video_ids += [ i18n.get_video_id(yid) for yid in youtube_ids_to_delete ] if len(video_files_to_delete): self.stdout.write( "Deleted %d VideoFile models (to mark them as not downloaded, since they were in a bad state)\n" % len(video_files_to_delete)) files = glob.glob(settings.CONTENT_ROOT + "*.mp4") videos_marked_at_all = set( [video.youtube_id for video in VideoFile.objects.all()]) videos_marked_as_in_progress = set([ video.youtube_id for video in VideoFile.objects.filter(download_in_progress=True) ]) videos_marked_as_unstarted = set([ video.youtube_id for video in VideoFile.objects.filter(percent_complete=0, download_in_progress=False) ]) videos_flagged_for_download = set([ video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True) ]) videos_in_filesystem = set([ path.replace("\\", "/").split("/")[-1].split(".")[0] for path in files ]) # Files that exist, but are not in the DB, should be assumed to be good videos, # and just needing to be added to the DB. Add them to the DB in this way, # so that these files also trigger the update code below (and trigger cache invalidation) video_ids_needing_model_creation = list(videos_in_filesystem - videos_marked_at_all) count = len(video_ids_needing_model_creation) if count: # OK to do bulk_create; cache invalidation triggered via save download VideoFile.objects.bulk_create([ VideoFile(youtube_id=id, percent_complete=0, download_in_progress=False) for id in video_ids_needing_model_creation ]) self.stdout.write( "Created %d VideoFile models (to mark them as complete, since the files exist)\n" % len(video_ids_needing_model_creation)) touched_video_ids += [ i18n.get_video_id(yid) or yid for yid in video_ids_needing_model_creation ] # Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False # These should be individually saved to be 100% complete, to trigger their availability (and cache invalidation) count = 0 for chunk in break_into_chunks(videos_in_filesystem): video_files_needing_model_update = VideoFile.objects.filter( percent_complete=0, download_in_progress=False, youtube_id__in=chunk) count += video_files_needing_model_update.count() for videofile in video_files_needing_model_update: videofile.percent_complete = 100 videofile.flagged_for_download = False videofile.save() if count: self.stdout.write( "Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % count) # VideoFile objects say they're available, but that don't actually exist. count = 0 videos_needing_model_deletion_chunked = break_into_chunks( videos_marked_at_all - videos_in_filesystem - videos_flagged_for_download) for chunk in videos_needing_model_deletion_chunked: video_files_needing_model_deletion = VideoFile.objects.filter( youtube_id__in=chunk) count += video_files_needing_model_deletion.count() video_files_needing_model_deletion.delete() touched_video_ids += [ i18n.get_video_id(yid) or yid for yid in chunk ] if count: self.stdout.write( "Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % count) if options["auto_cache"] and caching_enabled and touched_video_ids: caching.regenerate_all_pages_related_to_videos( video_ids=list(set(touched_video_ids)))
def handle(self, *args, **options): self.video = None handled_youtube_ids = [] # stored to deal with caching failed_youtube_ids = [] # stored to avoid requerying failures. set_process_priority.lowest(logging=settings.LOG) try: while True: # loop until the method is aborted # Grab any video that hasn't been tried yet videos = VideoFile.objects \ .filter(flagged_for_download=True, download_in_progress=False) \ .exclude(youtube_id__in=failed_youtube_ids) video_count = videos.count() if video_count == 0: self.stdout.write("Nothing to download; exiting.\n") break # Grab a video as OURS to handle, set fields to indicate to others that we're on it! # Update the video logging video = videos[0] video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) # Update the progress logging self.set_stages(num_stages=video_count + len(handled_youtube_ids) + len(failed_youtube_ids) + int(options["auto_cache"])) if not self.started(): self.start(stage_name=video.youtube_id) # Initiate the download process try: download_video(video.youtube_id, callback=partial(self.download_progress_callback, video)) handled_youtube_ids.append(video.youtube_id) self.stdout.write("Download is complete!\n") except Exception as e: # On error, report the error, mark the video as not downloaded, # and allow the loop to try other videos. self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e)) video.download_in_progress = False video.flagged_for_download = not isinstance(e, URLNotFound) # URLNotFound means, we won't try again video.save() # Rather than getting stuck on one video, continue to the next video. failed_youtube_ids.append(video.youtube_id) continue # This can take a long time, without any further update, so ... best to avoid. if options["auto_cache"] and caching.caching_is_enabled() and handled_youtube_ids: self.update_stage(stage_name=self.video.youtube_id, stage_percent=0, notes=_("Generating all pages related to videos.")) caching.regenerate_all_pages_related_to_videos(video_ids=list(set([i18n.get_video_id(yid) or yid for yid in handled_youtube_ids]))) # Update self.complete(notes=_("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully.") % { "num_handled_videos": len(handled_youtube_ids), "num_total_videos": len(handled_youtube_ids) + len(failed_youtube_ids), }) except Exception as e: sys.stderr.write("Error: %s\n" % e) self.cancel(notes=_("Error: %s") % e)