def start_subtitle_download(request):
new_only = simplejson.loads(request.raw_post_data
or "{}").get("new_only", False)
language = simplejson.loads(request.raw_post_data
or "{}").get("language", "")
language_list = topicdata.LANGUAGE_LIST
current_language = Settings.get("subtitle_language")
new_only = new_only and (current_language == language)
if language in language_list:
Settings.set("subtitle_language", language)
else:
return JsonResponse(
{
"error":
"This language is not currently supported - please update the language list"
},
status=500)
if new_only:
videofiles = VideoFile.objects.filter(Q(percent_complete=100)
| Q(flagged_for_download=True),
subtitles_downloaded=False)
else:
videofiles = VideoFile.objects.filter(
Q(percent_complete=100) | Q(flagged_for_download=True))
for videofile in videofiles:
videofile.cancel_download = False
if videofile.subtitle_download_in_progress:
continue
videofile.flagged_for_subtitle_download = True
if not new_only:
videofile.subtitles_downloaded = False
videofile.save()
force_job("subtitledownload", "Download Subtitles")
return JsonResponse({})
def handle(self, *args, **options):
while True: # loop until the method is aborted
if VideoFile.objects.filter(download_in_progress=True).count() > 0:
self.stderr.write("Another download is still in progress; aborting.\n")
return
videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False)
if videos.count() == 0:
self.stdout.write("Nothing to download; aborting.\n")
return
video = videos[0]
if video.cancel_download == True:
video.download_in_progress = False
video.save()
self.stdout.write("Download cancelled; aborting.\n")
return
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write("Downloading video '%s'...\n" % video.youtube_id)
try:
download_video(video.youtube_id, callback=download_progress_callback(self, video))
self.stdout.write("Download is complete!\n")
except Exception as e:
self.stderr.write("Error in downloading: %s\n" % e)
video.download_in_progress = False
video.save()
force_job("videodownload", "Download Videos")
return
def video_handler(request, video, prev=None, next=None):
video_exists = VideoFile.objects.filter(pk=video['youtube_id']).exists()
# If we detect that a video exists, but it's not on disk, then
# force the database to update. No race condition here for saving
# progress in a VideoLog: it is not dependent on VideoFile.
if not video_exists and topic_tools.is_video_on_disk(video['youtube_id']):
force_job("videoscan")
video_exists = True
if not video_exists:
if request.is_admin:
# TODO(bcipolli): add a link, with querystring args that auto-checks this video in the topic tree
messages.warning(request, _("This video was not found! You can download it by going to the Update page."))
elif request.is_logged_in:
messages.warning(request, _("This video was not found! Please contact your teacher or an admin to have it downloaded."))
elif not request.is_logged_in:
messages.warning(request, _("This video was not found! You must login as an admin/teacher to download the video."))
context = {
"video": video,
"title": video["title"],
"video_exists": video_exists,
"prev": prev,
"next": next,
}
return context
def start_video_download(request):
youtube_ids = OrderedSet(
simplejson.loads(request.raw_post_data or "{}").get("youtube_ids", []))
video_files_to_create = [
id for id in youtube_ids
if not get_object_or_None(VideoFile, youtube_id=id)
]
video_files_to_update = youtube_ids - OrderedSet(video_files_to_create)
VideoFile.objects.bulk_create([
VideoFile(youtube_id=id, flagged_for_download=True)
for id in video_files_to_create
])
for chunk in break_into_chunks(youtube_ids):
video_files_needing_model_update = VideoFile.objects.filter(
download_in_progress=False,
youtube_id__in=chunk).exclude(percent_complete=100)
video_files_needing_model_update.update(percent_complete=0,
cancel_download=False,
flagged_for_download=True)
force_job("videodownload", "Download Videos")
return JsonResponse({})
def start_subtitle_download(request):
update_set = simplejson.loads(request.raw_post_data or "{}").get("update_set", "existing")
language = simplejson.loads(request.raw_post_data or "{}").get("language", "")
# Set subtitle language
Settings.set("subtitle_language", language)
# Get the json file with all srts
request_url = "http://%s/static/data/subtitles/languages/%s_available_srts.json" % (settings.CENTRAL_SERVER_HOST, language)
try:
r = requests.get(request_url)
r.raise_for_status() # will return none if 200, otherwise will raise HTTP error
available_srts = set((r.json)["srt_files"])
except ConnectionError:
return JsonResponse({"error": "The central server is currently offline."}, status=500)
except HTTPError:
return JsonResponse({"error": "No subtitles available on central server for language code: %s; aborting." % language}, status=500)
if update_set == "existing":
videofiles = VideoFile.objects.filter(subtitles_downloaded=False, subtitle_download_in_progress=False)
else:
videofiles = VideoFile.objects.filter(subtitle_download_in_progress=False)
queue_count = 0
for chunk in break_into_chunks(available_srts):
queue_count += videofiles.filter(youtube_id__in=chunk).update(flagged_for_subtitle_download=True, subtitles_downloaded=False)
if queue_count == 0:
return JsonResponse({"info": "There aren't any subtitles available in this language for your currently downloaded videos."}, status=200)
force_job("subtitledownload", "Download Subtitles")
return JsonResponse({})
def video_handler(request, video, prev=None, next=None):
video_exists = VideoFile.objects.filter(pk=video['youtube_id']).exists()
# If we detect that a video exists, but it's not on disk, then
# force the database to update. No race condition here for saving
# progress in a VideoLog: it is not dependent on VideoFile.
if not video_exists and topic_tools.is_video_on_disk(video['youtube_id']):
force_job("videoscan")
video_exists = True
if not video_exists:
if request.is_admin:
# TODO(bcipolli): add a link, with querystring args that auto-checks this video in the topic tree
messages.warning(
request,
_("This video was not found! You can download it by going to the Update page."
))
elif request.is_logged_in:
messages.warning(
request,
_("This video was not found! Please contact your teacher or an admin to have it downloaded."
))
elif not request.is_logged_in:
messages.warning(
request,
_("This video was not found! You must login as an admin/teacher to download the video."
))
context = {
"video": video,
"title": video["title"],
"video_exists": video_exists,
"prev": prev,
"next": next,
}
return context
def start_video_download(request):
youtube_ids = OrderedSet(simplejson.loads(request.raw_post_data or "{}").get("youtube_ids", []))
video_files_to_create = [id for id in youtube_ids if not get_object_or_None(VideoFile, youtube_id=id)]
video_files_to_update = youtube_ids - OrderedSet(video_files_to_create)
VideoFile.objects.bulk_create([VideoFile(youtube_id=id, flagged_for_download=True) for id in video_files_to_create])
for chunk in break_into_chunks(video_files_to_create):
video_files_needing_model_update = VideoFile.objects.filter(download_in_progress=False, youtube_id__in=chunk)
video_files_needing_model_update.update(percent_complete=0, cancel_download=False, flagged_for_download=True)
force_job("videodownload", "Download Videos")
return JsonResponse({})
def cancel_downloads(request):
# clear all download in progress flags, to make sure new downloads will go through
VideoFile.objects.all().update(download_in_progress=False)
# unflag all video downloads
VideoFile.objects.filter(flagged_for_download=True).update(cancel_download=True, flagged_for_download=False)
# unflag all subtitle downloads
VideoFile.objects.filter(flagged_for_subtitle_download=True).update(cancel_download=True, flagged_for_subtitle_download=False)
force_job("videodownload", stop=True)
force_job("subtitledownload", stop=True)
return JsonResponse({})
def cancel_downloads(request):
# clear all download in progress flags, to make sure new downloads will go through
VideoFile.objects.all().update(download_in_progress=False)
# unflag all video downloads
VideoFile.objects.filter(flagged_for_download=True).update(cancel_download=True, flagged_for_download=False)
# unflag all subtitle downloads
VideoFile.objects.filter(flagged_for_subtitle_download=True).update(cancel_download=True, flagged_for_subtitle_download=False)
force_job("videodownload", stop=True)
force_job("subtitledownload", stop=True)
return JsonResponse({})
def update(request):
call_command("videoscan")
force_job("videodownload", "Download Videos")
force_job("subtitledownload", "Download Subtitles")
language_lookup = topicdata.LANGUAGE_LOOKUP
language_list = topicdata.LANGUAGE_LIST
default_language = Settings.get("subtitle_language") or "en"
if default_language not in language_list:
language_list.append(default_language)
languages = [{"id": key, "name": language_lookup[key]} for key in language_list]
languages = sorted(languages, key=lambda k: k["name"])
context = {
"languages": languages,
"default_language": default_language,
}
return context
def update(request):
call_command("videoscan")
force_job("videodownload", "Download Videos")
force_job("subtitledownload", "Download Subtitles")
language_lookup = topicdata.LANGUAGE_LOOKUP
language_list = topicdata.LANGUAGE_LIST
default_language = Settings.get("subtitle_language") or "en"
if default_language not in language_list:
language_list.append(default_language)
languages = [{"id": key, "name": language_lookup[key]} for key in language_list]
languages = sorted(languages, key=lambda k: k["name"])
context = {
"languages": languages,
"default_language": default_language,
}
return context
def handle(self, *args, **options):
caching_enabled = settings.CACHE_TIME != 0
handled_video_ids = []
while True: # loop until the method is aborted
if VideoFile.objects.filter(download_in_progress=True).count() > 0:
self.stderr.write("Another download is still in progress; aborting.\n")
break
videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False)
if videos.count() == 0:
self.stdout.write("Nothing to download; aborting.\n")
break
video = videos[0]
if video.cancel_download == True:
video.download_in_progress = False
video.save()
self.stdout.write("Download cancelled; aborting.\n")
break
video.download_in_progress = True
video.percent_complete = 0
video.save()
self.stdout.write("Downloading video '%s'...\n" % video.youtube_id)
try:
download_video(video.youtube_id, callback=download_progress_callback(self, video))
self.stdout.write("Download is complete!\n")
except Exception as e:
self.stderr.write("Error in downloading: %s\n" % e)
video.download_in_progress = False
video.save()
force_job("videodownload", "Download Videos") # infinite recursive call? :(
break
handled_video_ids.append(video.youtube_id)
# Expire, but don't regenerate until the very end, for efficiency.
if caching_enabled:
caching.invalidate_all_pages_related_to_video(video_id=video.youtube_id)
if options["auto_cache"] and caching_enabled and handled_video_ids:
caching.regenerate_all_pages_related_to_videos(video_ids=handled_video_ids)
def handle(self, *args, **options):
language = Settings.get("subtitle_language")
while True: # loop until the method is aborted
if VideoFile.objects.filter(
subtitle_download_in_progress=True).count() > 4:
self.stderr.write(
"Maximum downloads are in progress; aborting.\n")
return
videos = VideoFile.objects.filter(
flagged_for_subtitle_download=True,
subtitle_download_in_progress=False)
if videos.count() == 0:
self.stdout.write("Nothing to download; aborting.\n")
return
video = videos[0]
video.subtitle_download_in_progress = True
video.save()
self.stdout.write("Downloading subtitles for video '%s'...\n" %
video.youtube_id)
try:
download_subtitles(video.youtube_id, language)
self.stdout.write("Download is complete!\n")
video.subtitles_downloaded = True
video.subtitle_download_in_progress = False
video.flagged_for_subtitle_download = False
video.save()
except NoSubs as e:
video.flagged_for_subtitle_download = False
video.subtitle_download_in_progress = False
video.subtitles_downloaded = True
video.save()
self.stderr.write("No subtitles available\n")
except Exception as e:
self.stderr.write("Error in downloading subtitles: %s\n" % e)
video.subtitle_download_in_progress = False
video.flagged_for_subtitle_download = False
video.save()
force_job("subtitledownload", "Download Subtitles")
return
def update(request):
call_command("videoscan") # Could potentially be very slow, blocking request.
force_job("videodownload", "Download Videos")
force_job("subtitledownload", "Download Subtitles")
default_language = Settings.get("subtitle_language") or "en"
device = Device.get_own_device()
zone = device.get_zone()
context = {
"default_language": default_language,
"registered": Settings.get("registered"),
"zone_id": zone.id if zone else None,
"device_id": device.id,
"video_count": VideoFile.objects.filter(percent_complete=100).count(),
}
return context
def update(request):
call_command(
"videoscan") # Could potentially be very slow, blocking request.
force_job("videodownload", "Download Videos")
force_job("subtitledownload", "Download Subtitles")
default_language = Settings.get("subtitle_language") or "en"
device = Device.get_own_device()
zone = device.get_zone()
context = {
"default_language": default_language,
"registered": Settings.get("registered"),
"zone_id": zone.id if zone else None,
"device_id": device.id,
"video_count": VideoFile.objects.filter(percent_complete=100).count(),
}
return context
def start_subtitle_download(request):
update_set = simplejson.loads(request.raw_post_data or "{}").get("update_set", "existing")
language = simplejson.loads(request.raw_post_data or "{}").get("language", "")
language_list = topicdata.LANGUAGE_LIST
language_lookup = topicdata.LANGUAGE_LOOKUP
# Reset the language
current_language = Settings.get("subtitle_language")
if language in language_list:
Settings.set("subtitle_language", language)
else:
return JsonResponse({"error": "This language is not currently supported - please update the language list"}, status=500)
language_name = language_lookup.get(language)
# Get the json file with all srts
request_url = "http://%s/static/data/subtitles/languages/%s_available_srts.json" % (settings.CENTRAL_SERVER_HOST, language)
try:
r = requests.get(request_url)
r.raise_for_status() # will return none if 200, otherwise will raise HTTP error
available_srts = set((r.json)["srt_files"])
except ConnectionError:
return JsonResponse({"error": "The central server is currently offline."}, status=500)
except HTTPError:
return JsonResponse({"error": "No subtitles available on central server for %s (language code: %s); aborting." % (language_name, language)}, status=500)
if update_set == "existing":
videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), subtitles_downloaded=False, youtube_id__in=available_srts)
else:
videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), youtube_id__in=available_srts)
if not videofiles:
return JsonResponse({"info": "There aren't any subtitles available in %s (language code: %s) for your current videos." % (language_name, language)}, status=200)
else:
for videofile in videofiles:
videofile.cancel_download = False
if videofile.subtitle_download_in_progress:
continue
videofile.flagged_for_subtitle_download = True
if update_set == "all":
videofile.subtitles_downloaded = False
videofile.save()
force_job("subtitledownload", "Download Subtitles")
return JsonResponse({})
def handle(self, *args, **options):
language = Settings.get("subtitle_language")
while True: # loop until the method is aborted
if VideoFile.objects.filter(subtitle_download_in_progress=True).count() > 4:
self.stderr.write("Maximum downloads are in progress; aborting.\n")
return
videos = VideoFile.objects.filter(flagged_for_subtitle_download=True, subtitle_download_in_progress=False)
if videos.count() == 0:
self.stdout.write("Nothing to download; aborting.\n")
return
video = videos[0]
video.subtitle_download_in_progress = True
video.save()
self.stdout.write("Downloading subtitles for video '%s'... " % video.youtube_id)
self.stdout.flush()
try:
download_subtitles(video.youtube_id, language)
self.stdout.write("Download is complete!\n")
video.subtitles_downloaded = True
video.subtitle_download_in_progress = False
video.flagged_for_subtitle_download = False
video.save()
except NoSubs as e:
video.flagged_for_subtitle_download = False
video.subtitle_download_in_progress = False
video.subtitles_downloaded = True
self.stdout.write("\n");
video.save()
self.stderr.write("No subtitles available\n")
except Exception as e:
self.stderr.write("Error in downloading subtitles: %s\n" % e)
video.subtitle_download_in_progress = False
video.flagged_for_subtitle_download = False
video.save()
force_job("subtitledownload", "Download Subtitles")
return
def start_subtitle_download(request):
new_only = simplejson.loads(request.raw_post_data or "{}").get("new_only", False)
language = simplejson.loads(request.raw_post_data or "{}").get("language", "")
language_list = topicdata.LANGUAGE_LIST
current_language = Settings.get("subtitle_language")
new_only = new_only and (current_language == language)
if language in language_list:
Settings.set("subtitle_language", language)
else:
return JsonResponse({"error": "This language is not currently supported - please update the language list"}, status=500)
if new_only:
videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), subtitles_downloaded=False)
else:
videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True))
for videofile in videofiles:
videofile.cancel_download = False
if videofile.subtitle_download_in_progress:
continue
videofile.flagged_for_subtitle_download = True
if not new_only:
videofile.subtitles_downloaded = False
videofile.save()
force_job("subtitledownload", "Download Subtitles")
return JsonResponse({})
def set_as_registered():
force_job("syncmodels", "Secure Sync",
"HOURLY") # now launches asynchronously
Settings.set("registered", True)
def retry_video_download(request):
"""Clear any video still accidentally marked as in-progress, and restart the download job.
"""
VideoFile.objects.filter(download_in_progress=True).update(download_in_progress=False, percent_complete=0)
force_job("videodownload", "Download Videos")
return JsonResponse({})
def set_as_registered():
force_job("syncmodels", "Secure Sync", "HOURLY") # now launches asynchronously
Settings.set("registered", True)
def set_as_registered():
force_job("syncmodels", "Secure Sync", "HOURLY")
Settings.set("registered", True)
def cancel_downloads(request):
videofiles = VideoFile.objects.filter(Q(flagged_for_download=True) | Q(flagged_for_subtitle_download=True))
videofiles.update(cancel_download=True, flagged_for_download=False, flagged_for_subtitle_download=False)
force_job("videodownload", stop=True)
force_job("subtitledownload", stop=True)
return JsonResponse({})
def set_as_registered():
force_job("syncmodels", "Secure Sync", "HOURLY")
Settings.set("registered", True)
