def handle(self, *args, **options):
self.video = None
handled_youtube_ids = [] # stored to deal with caching
failed_youtube_ids = [] # stored to avoid requerying failures.
set_process_priority.lowest(logging=settings.LOG)
try:
while True: # loop until the method is aborted
# Grab any video that hasn't been tried yet
videos = VideoFile.objects \
.filter(flagged_for_download=True, download_in_progress=False) \
.exclude(youtube_id__in=failed_youtube_ids)
video_count = videos.count()
if video_count == 0:
self.stdout.write("Nothing to download; exiting.\n")
break
# Grab a video as OURS to handle, set fields to indicate to others that we're on it!
# Update the video logging
video = videos[0]
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write("Downloading video '%s'...\n" % video.youtube_id)
# Update the progress logging
self.set_stages(num_stages=video_count + len(handled_youtube_ids) + len(failed_youtube_ids) + int(options["auto_cache"]))
if not self.started():
self.start(stage_name=video.youtube_id)
# Initiate the download process
try:
download_video(video.youtube_id, callback=partial(self.download_progress_callback, video))
handled_youtube_ids.append(video.youtube_id)
self.stdout.write("Download is complete!\n")
except Exception as e:
# On error, report the error, mark the video as not downloaded,
# and allow the loop to try other videos.
self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e))
video.download_in_progress = False
video.flagged_for_download = not isinstance(e, URLNotFound) # URLNotFound means, we won't try again
video.save()
# Rather than getting stuck on one video, continue to the next video.
failed_youtube_ids.append(video.youtube_id)
continue
# This can take a long time, without any further update, so ... best to avoid.
if options["auto_cache"] and caching.caching_is_enabled() and handled_youtube_ids:
self.update_stage(stage_name=self.video.youtube_id, stage_percent=0, notes=_("Generating all pages related to videos."))
caching.regenerate_all_pages_related_to_videos(video_ids=list(set([i18n.get_video_id(yid) or yid for yid in handled_youtube_ids])))
# Update
self.complete(notes=_("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully.") % {
"num_handled_videos": len(handled_youtube_ids),
"num_total_videos": len(handled_youtube_ids) + len(failed_youtube_ids),
})
except Exception as e:
sys.stderr.write("Error: %s\n" % e)
self.cancel(notes=_("Error: %s") % e)
def handle(self, *args, **options):
self.video = None
handled_youtube_ids = [] # stored to deal with caching
failed_youtube_ids = [] # stored to avoid requerying failures.
set_process_priority.lowest(logging=settings.LOG)
try:
while True: # loop until the method is aborted
# Grab any video that hasn't been tried yet
videos = VideoFile.objects \
.filter(flagged_for_download=True, download_in_progress=False) \
.exclude(youtube_id__in=failed_youtube_ids)
video_count = videos.count()
if video_count == 0:
self.stdout.write("Nothing to download; exiting.\n")
break
# Grab a video as OURS to handle, set fields to indicate to others that we're on it!
# Update the video logging
video = videos[0]
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write("Downloading video '%s'...\n" %
video.youtube_id)
# Update the progress logging
self.set_stages(
num_stages=video_count + len(handled_youtube_ids) +
len(failed_youtube_ids) + int(options["auto_cache"]))
if not self.started():
self.start(stage_name=video.youtube_id)
# Initiate the download process
try:
download_video(video.youtube_id,
callback=partial(
self.download_progress_callback, video))
handled_youtube_ids.append(video.youtube_id)
self.stdout.write("Download is complete!\n")
except Exception as e:
# On error, report the error, mark the video as not downloaded,
# and allow the loop to try other videos.
self.stderr.write("Error in downloading %s: %s\n" %
(video.youtube_id, e))
video.download_in_progress = False
video.flagged_for_download = not isinstance(
e,
URLNotFound) # URLNotFound means, we won't try again
video.save()
# Rather than getting stuck on one video, continue to the next video.
failed_youtube_ids.append(video.youtube_id)
continue
# This can take a long time, without any further update, so ... best to avoid.
if options["auto_cache"] and caching.caching_is_enabled(
) and handled_youtube_ids:
self.update_stage(
stage_name=self.video.youtube_id,
stage_percent=0,
notes=_("Generating all pages related to videos."))
caching.regenerate_all_pages_related_to_videos(video_ids=list(
set([
i18n.get_video_id(yid) or yid
for yid in handled_youtube_ids
])))
# Update
self.complete(
notes=
_("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully."
) % {
"num_handled_videos":
len(handled_youtube_ids),
"num_total_videos":
len(handled_youtube_ids) + len(failed_youtube_ids),
})
except Exception as e:
sys.stderr.write("Error: %s\n" % e)
self.cancel(notes=_("Error: %s") % e)
def handle(self, *args, **options):
caching_enabled = settings.CACHE_TIME != 0
handled_video_ids = [] # stored to deal with caching
while True: # loop until the method is aborted
if VideoFile.objects.filter(download_in_progress=True).count() > 0:
self.stderr.write("Another download is still in progress; aborting.\n")
break
# Grab any video that hasn't been tried yet
videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False)
if videos.count() == 0:
self.stdout.write("Nothing to download; aborting.\n")
break
video = videos[0]
# User intervention
if video.cancel_download == True:
video.download_in_progress = False
video.save()
self.stdout.write("Download cancelled; aborting.\n")
break
# Grab a video as OURS to handle, set fields to indicate to others that we're on it!
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write("Downloading video '%s'...\n" % video.youtube_id)
try:
download_video(video.youtube_id, callback=download_progress_callback(self, video))
handled_video_ids.append(video.youtube_id)
self.stdout.write("Download is complete!\n")
except Exception as e:
if isinstance(e, URLNotFound):
# This should never happen, but if it does, remove the VideoFile from the queue, and continue
# to the next video. Warning: this will leave the update page in a weird state, currently
# (and require a refresh of the update page in order to start showing progress again)
video.delete()
continue
# On connection error, report the error, mark the video as not downloaded, and give up for now.
self.stderr.write("Error in downloading %s: %s\n" % (video.youtube_id, e))
video.download_in_progress = False
video.percent_complete = 0
video.save()
break
# Expire, but don't regenerate until the very end, for efficiency.
if caching_enabled:
caching.invalidate_all_pages_related_to_video(video_id=video.youtube_id)
# After all is done, regenerate all pages
# since this is computationally intensive, only do it after we're sure
# nothing more will change (so that we don't regenerate something that is
# later invalidated by another video downloaded in the loop)
if options["auto_cache"] and caching_enabled and handled_video_ids:
caching.regenerate_all_pages_related_to_videos(video_ids=handled_video_ids)