def reinitialize_server(self):
"""Reset the server state."""
# Next, call videoscan.
logging.info("Running videoscan.")
call_command("videoscan")
# Finally, pre-load global data
initialize_content_caches()
def reinitialize_server(self):
"""Reset the server state."""
# Next, call videoscan.
logging.info("Running videoscan.")
call_command("videoscan")
# Finally, pre-load global data
initialize_content_caches()
def reinitialize_server(self):
"""Reset the server state."""
logging.info("Invalidating the web cache.")
from fle_utils.internet.webcache import invalidate_web_cache
invalidate_web_cache()
# Next, call videoscan.
logging.info("Running videoscan.")
call_command("videoscan")
# Finally, pre-load global data
initialize_content_caches()
def delete_videos(request):
"""
API endpoint for deleting videos.
"""
youtube_ids = simplejson.loads(request.body or "{}").get("youtube_ids", [])
num_deleted = 0
for id in youtube_ids:
# Delete the file on disk
delete_downloaded_files(id)
# Delete the file in the database
found_videos = VideoFile.objects.filter(youtube_id=id)
num_deleted += found_videos.count()
found_videos.delete()
initialize_content_caches(force=True)
return JsonResponseMessageSuccess(_("Deleted %(num_videos)s video(s) successfully.") % {"num_videos": num_deleted})
def delete_videos(request):
"""
API endpoint for deleting videos.
"""
youtube_ids = simplejson.loads(request.body or "{}").get("youtube_ids", [])
num_deleted = 0
for id in youtube_ids:
# Delete the file on disk
delete_downloaded_files(id)
# Delete the file in the database
found_videos = VideoFile.objects.filter(youtube_id=id)
num_deleted += found_videos.count()
found_videos.delete()
initialize_content_caches(force=True)
return JsonResponseMessageSuccess(
_("Deleted %(num_videos)s video(s) successfully.") %
{"num_videos": num_deleted})
def setUp(self):
super(UpdatesTestCase, self).setUp()
# Set up the topic tree
initialize_content_caches(force=True)
def handle(self, *args, **options):
self.video = None
handled_youtube_ids = [] # stored to deal with caching
failed_youtube_ids = [] # stored to avoid requerying failures.
set_process_priority.lowest(logging=settings.LOG)
try:
while True: # loop until the method is aborted
# Grab any video that hasn't been tried yet
videos = VideoFile.objects \
.filter(flagged_for_download=True, download_in_progress=False) \
.exclude(youtube_id__in=failed_youtube_ids)
video_count = videos.count()
if video_count == 0:
self.stdout.write(_("Nothing to download; exiting.") + "\n")
break
# Grab a video as OURS to handle, set fields to indicate to others that we're on it!
# Update the video logging
video = videos[0]
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write((_("Downloading video '%(youtube_id)s'...") + "\n") % {"youtube_id": video.youtube_id})
# Update the progress logging
self.set_stages(num_stages=video_count + len(handled_youtube_ids) + len(failed_youtube_ids) + int(options["auto_cache"]))
if not self.started():
self.start(stage_name=video.youtube_id)
# Initiate the download process
try:
ensure_dir(settings.CONTENT_ROOT)
progress_callback = partial(self.download_progress_callback, video)
try:
# Download via urllib
download_video(video.youtube_id, callback=progress_callback)
except URLNotFound:
# Video was not found on amazon cloud service,
# either due to a KA mistake, or due to the fact
# that it's a dubbed video.
#
# We can use youtube-dl to get that video!!
logging.debug(_("Retrieving youtube video %(youtube_id)s via youtube-dl") % {"youtube_id": video.youtube_id})
def youtube_dl_cb(stats, progress_callback, *args, **kwargs):
if stats['status'] == "finished":
percent = 100.
elif stats['status'] == "downloading":
percent = 100. * stats['downloaded_bytes'] / stats['total_bytes']
else:
percent = 0.
progress_callback(percent=percent)
scrape_video(video.youtube_id, quiet=not settings.DEBUG, callback=partial(youtube_dl_cb, progress_callback=progress_callback))
except IOError as e:
logging.exception(e)
video.download_in_progress = False
video.save()
failed_youtube_ids.append(video.youtube_id)
time.sleep(10)
continue
# If we got here, we downloaded ... somehow :)
handled_youtube_ids.append(video.youtube_id)
self.stdout.write(_("Download is complete!") + "\n")
except DownloadCancelled:
# Cancellation event
video.percent_complete = 0
video.flagged_for_download = False
video.download_in_progress = False
video.save()
failed_youtube_ids.append(video.youtube_id)
except Exception as e:
# On error, report the error, mark the video as not downloaded,
# and allow the loop to try other videos.
msg = _("Error in downloading %(youtube_id)s: %(error_msg)s") % {"youtube_id": video.youtube_id, "error_msg": unicode(e)}
self.stderr.write("%s\n" % msg)
# If a connection error, we should retry.
if isinstance(e, DownloadError):
connection_error = "[Errno 8]" in e.args[0]
elif isinstance(e, IOError) and hasattr(e, "strerror"):
connection_error = e.strerror[0] == 8
else:
connection_error = False
video.download_in_progress = False
video.flagged_for_download = connection_error # Any error other than a connection error is fatal.
video.save()
# Rather than getting stuck on one video, continue to the next video.
self.update_stage(stage_status="error", notes=_("%(error_msg)s; continuing to next video.") % {"error_msg": msg})
failed_youtube_ids.append(video.youtube_id)
continue
# Update
self.complete(notes=_("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully.") % {
"num_handled_videos": len(handled_youtube_ids),
"num_total_videos": len(handled_youtube_ids) + len(failed_youtube_ids),
})
caching.initialize_content_caches()
except Exception as e:
self.cancel(stage_status="error", notes=_("Error: %(error_msg)s") % {"error_msg": e})
raise
def handle(self, *args, **options):
self.video = None
handled_youtube_ids = [] # stored to deal with caching
failed_youtube_ids = [] # stored to avoid requerying failures.
set_process_priority.lowest(logging=settings.LOG)
try:
while True: # loop until the method is aborted
# Grab any video that hasn't been tried yet
videos = VideoFile.objects \
.filter(flagged_for_download=True, download_in_progress=False) \
.exclude(youtube_id__in=failed_youtube_ids)
video_count = videos.count()
if video_count == 0:
self.stdout.write(
_("Nothing to download; exiting.") + "\n")
break
# Grab a video as OURS to handle, set fields to indicate to others that we're on it!
# Update the video logging
video = videos[0]
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write(
(_("Downloading video '%(youtube_id)s'...") + "\n") %
{"youtube_id": video.youtube_id})
# Update the progress logging
self.set_stages(
num_stages=video_count + len(handled_youtube_ids) +
len(failed_youtube_ids) + int(options["auto_cache"]))
if not self.started():
self.start(stage_name=video.youtube_id)
# Initiate the download process
try:
ensure_dir(settings.CONTENT_ROOT)
progress_callback = partial(
self.download_progress_callback, video)
try:
# Download via urllib
download_video(video.youtube_id,
callback=progress_callback)
except URLNotFound:
# Video was not found on amazon cloud service,
# either due to a KA mistake, or due to the fact
# that it's a dubbed video.
#
# We can use youtube-dl to get that video!!
logging.debug(
_("Retrieving youtube video %(youtube_id)s via youtube-dl"
) % {"youtube_id": video.youtube_id})
def youtube_dl_cb(stats, progress_callback, *args,
**kwargs):
if stats['status'] == "finished":
percent = 100.
elif stats['status'] == "downloading":
percent = 100. * stats[
'downloaded_bytes'] / stats['total_bytes']
else:
percent = 0.
progress_callback(percent=percent)
scrape_video(video.youtube_id,
quiet=not settings.DEBUG,
callback=partial(
youtube_dl_cb,
progress_callback=progress_callback))
except IOError as e:
logging.exception(e)
video.download_in_progress = False
video.save()
failed_youtube_ids.append(video.youtube_id)
time.sleep(10)
continue
# If we got here, we downloaded ... somehow :)
handled_youtube_ids.append(video.youtube_id)
self.stdout.write(_("Download is complete!") + "\n")
except DownloadCancelled:
# Cancellation event
video.percent_complete = 0
video.flagged_for_download = False
video.download_in_progress = False
video.save()
failed_youtube_ids.append(video.youtube_id)
except Exception as e:
# On error, report the error, mark the video as not downloaded,
# and allow the loop to try other videos.
msg = _(
"Error in downloading %(youtube_id)s: %(error_msg)s"
) % {
"youtube_id": video.youtube_id,
"error_msg": unicode(e)
}
self.stderr.write("%s\n" % msg)
# If a connection error, we should retry.
if isinstance(e, DownloadError):
connection_error = "[Errno 8]" in e.args[0]
elif isinstance(e, IOError) and hasattr(e, "strerror"):
connection_error = e.strerror[0] == 8
else:
connection_error = False
video.download_in_progress = False
video.flagged_for_download = connection_error # Any error other than a connection error is fatal.
video.save()
# Rather than getting stuck on one video, continue to the next video.
self.update_stage(
stage_status="error",
notes=_("%(error_msg)s; continuing to next video.") %
{"error_msg": msg})
failed_youtube_ids.append(video.youtube_id)
continue
# Update
self.complete(
notes=
_("Downloaded %(num_handled_videos)s of %(num_total_videos)s videos successfully."
) % {
"num_handled_videos":
len(handled_youtube_ids),
"num_total_videos":
len(handled_youtube_ids) + len(failed_youtube_ids),
})
caching.initialize_content_caches()
except Exception as e:
self.cancel(stage_status="error",
notes=_("Error: %(error_msg)s") % {"error_msg": e})
raise