def start_subtitle_download(request): new_only = simplejson.loads(request.raw_post_data or "{}").get("new_only", False) language = simplejson.loads(request.raw_post_data or "{}").get("language", "") language_list = topicdata.LANGUAGE_LIST current_language = Settings.get("subtitle_language") new_only = new_only and (current_language == language) if language in language_list: Settings.set("subtitle_language", language) else: return JsonResponse( { "error": "This language is not currently supported - please update the language list" }, status=500) if new_only: videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), subtitles_downloaded=False) else: videofiles = VideoFile.objects.filter( Q(percent_complete=100) | Q(flagged_for_download=True)) for videofile in videofiles: videofile.cancel_download = False if videofile.subtitle_download_in_progress: continue videofile.flagged_for_subtitle_download = True if not new_only: videofile.subtitles_downloaded = False videofile.save() force_job("subtitledownload", "Download Subtitles") return JsonResponse({})
def handle(self, *args, **options): while True: # loop until the method is aborted if VideoFile.objects.filter(download_in_progress=True).count() > 0: self.stderr.write("Another download is still in progress; aborting.\n") return videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") return video = videos[0] if video.cancel_download == True: video.download_in_progress = False video.save() self.stdout.write("Download cancelled; aborting.\n") return video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) try: download_video(video.youtube_id, callback=download_progress_callback(self, video)) self.stdout.write("Download is complete!\n") except Exception as e: self.stderr.write("Error in downloading: %s\n" % e) video.download_in_progress = False video.save() force_job("videodownload", "Download Videos") return
def video_handler(request, video, prev=None, next=None): video_exists = VideoFile.objects.filter(pk=video['youtube_id']).exists() # If we detect that a video exists, but it's not on disk, then # force the database to update. No race condition here for saving # progress in a VideoLog: it is not dependent on VideoFile. if not video_exists and topic_tools.is_video_on_disk(video['youtube_id']): force_job("videoscan") video_exists = True if not video_exists: if request.is_admin: # TODO(bcipolli): add a link, with querystring args that auto-checks this video in the topic tree messages.warning(request, _("This video was not found! You can download it by going to the Update page.")) elif request.is_logged_in: messages.warning(request, _("This video was not found! Please contact your teacher or an admin to have it downloaded.")) elif not request.is_logged_in: messages.warning(request, _("This video was not found! You must login as an admin/teacher to download the video.")) context = { "video": video, "title": video["title"], "video_exists": video_exists, "prev": prev, "next": next, } return context
def start_video_download(request): youtube_ids = OrderedSet( simplejson.loads(request.raw_post_data or "{}").get("youtube_ids", [])) video_files_to_create = [ id for id in youtube_ids if not get_object_or_None(VideoFile, youtube_id=id) ] video_files_to_update = youtube_ids - OrderedSet(video_files_to_create) VideoFile.objects.bulk_create([ VideoFile(youtube_id=id, flagged_for_download=True) for id in video_files_to_create ]) for chunk in break_into_chunks(youtube_ids): video_files_needing_model_update = VideoFile.objects.filter( download_in_progress=False, youtube_id__in=chunk).exclude(percent_complete=100) video_files_needing_model_update.update(percent_complete=0, cancel_download=False, flagged_for_download=True) force_job("videodownload", "Download Videos") return JsonResponse({})
def start_subtitle_download(request): update_set = simplejson.loads(request.raw_post_data or "{}").get("update_set", "existing") language = simplejson.loads(request.raw_post_data or "{}").get("language", "") # Set subtitle language Settings.set("subtitle_language", language) # Get the json file with all srts request_url = "http://%s/static/data/subtitles/languages/%s_available_srts.json" % (settings.CENTRAL_SERVER_HOST, language) try: r = requests.get(request_url) r.raise_for_status() # will return none if 200, otherwise will raise HTTP error available_srts = set((r.json)["srt_files"]) except ConnectionError: return JsonResponse({"error": "The central server is currently offline."}, status=500) except HTTPError: return JsonResponse({"error": "No subtitles available on central server for language code: %s; aborting." % language}, status=500) if update_set == "existing": videofiles = VideoFile.objects.filter(subtitles_downloaded=False, subtitle_download_in_progress=False) else: videofiles = VideoFile.objects.filter(subtitle_download_in_progress=False) queue_count = 0 for chunk in break_into_chunks(available_srts): queue_count += videofiles.filter(youtube_id__in=chunk).update(flagged_for_subtitle_download=True, subtitles_downloaded=False) if queue_count == 0: return JsonResponse({"info": "There aren't any subtitles available in this language for your currently downloaded videos."}, status=200) force_job("subtitledownload", "Download Subtitles") return JsonResponse({})
def video_handler(request, video, prev=None, next=None): video_exists = VideoFile.objects.filter(pk=video['youtube_id']).exists() # If we detect that a video exists, but it's not on disk, then # force the database to update. No race condition here for saving # progress in a VideoLog: it is not dependent on VideoFile. if not video_exists and topic_tools.is_video_on_disk(video['youtube_id']): force_job("videoscan") video_exists = True if not video_exists: if request.is_admin: # TODO(bcipolli): add a link, with querystring args that auto-checks this video in the topic tree messages.warning( request, _("This video was not found! You can download it by going to the Update page." )) elif request.is_logged_in: messages.warning( request, _("This video was not found! Please contact your teacher or an admin to have it downloaded." )) elif not request.is_logged_in: messages.warning( request, _("This video was not found! You must login as an admin/teacher to download the video." )) context = { "video": video, "title": video["title"], "video_exists": video_exists, "prev": prev, "next": next, } return context
def start_video_download(request): youtube_ids = OrderedSet(simplejson.loads(request.raw_post_data or "{}").get("youtube_ids", [])) video_files_to_create = [id for id in youtube_ids if not get_object_or_None(VideoFile, youtube_id=id)] video_files_to_update = youtube_ids - OrderedSet(video_files_to_create) VideoFile.objects.bulk_create([VideoFile(youtube_id=id, flagged_for_download=True) for id in video_files_to_create]) for chunk in break_into_chunks(video_files_to_create): video_files_needing_model_update = VideoFile.objects.filter(download_in_progress=False, youtube_id__in=chunk) video_files_needing_model_update.update(percent_complete=0, cancel_download=False, flagged_for_download=True) force_job("videodownload", "Download Videos") return JsonResponse({})
def cancel_downloads(request): # clear all download in progress flags, to make sure new downloads will go through VideoFile.objects.all().update(download_in_progress=False) # unflag all video downloads VideoFile.objects.filter(flagged_for_download=True).update(cancel_download=True, flagged_for_download=False) # unflag all subtitle downloads VideoFile.objects.filter(flagged_for_subtitle_download=True).update(cancel_download=True, flagged_for_subtitle_download=False) force_job("videodownload", stop=True) force_job("subtitledownload", stop=True) return JsonResponse({})
def update(request): call_command("videoscan") force_job("videodownload", "Download Videos") force_job("subtitledownload", "Download Subtitles") language_lookup = topicdata.LANGUAGE_LOOKUP language_list = topicdata.LANGUAGE_LIST default_language = Settings.get("subtitle_language") or "en" if default_language not in language_list: language_list.append(default_language) languages = [{"id": key, "name": language_lookup[key]} for key in language_list] languages = sorted(languages, key=lambda k: k["name"]) context = { "languages": languages, "default_language": default_language, } return context
def handle(self, *args, **options): caching_enabled = settings.CACHE_TIME != 0 handled_video_ids = [] while True: # loop until the method is aborted if VideoFile.objects.filter(download_in_progress=True).count() > 0: self.stderr.write("Another download is still in progress; aborting.\n") break videos = VideoFile.objects.filter(flagged_for_download=True, download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") break video = videos[0] if video.cancel_download == True: video.download_in_progress = False video.save() self.stdout.write("Download cancelled; aborting.\n") break video.download_in_progress = True video.percent_complete = 0 video.save() self.stdout.write("Downloading video '%s'...\n" % video.youtube_id) try: download_video(video.youtube_id, callback=download_progress_callback(self, video)) self.stdout.write("Download is complete!\n") except Exception as e: self.stderr.write("Error in downloading: %s\n" % e) video.download_in_progress = False video.save() force_job("videodownload", "Download Videos") # infinite recursive call? :( break handled_video_ids.append(video.youtube_id) # Expire, but don't regenerate until the very end, for efficiency. if caching_enabled: caching.invalidate_all_pages_related_to_video(video_id=video.youtube_id) if options["auto_cache"] and caching_enabled and handled_video_ids: caching.regenerate_all_pages_related_to_videos(video_ids=handled_video_ids)
def handle(self, *args, **options): language = Settings.get("subtitle_language") while True: # loop until the method is aborted if VideoFile.objects.filter( subtitle_download_in_progress=True).count() > 4: self.stderr.write( "Maximum downloads are in progress; aborting.\n") return videos = VideoFile.objects.filter( flagged_for_subtitle_download=True, subtitle_download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") return video = videos[0] video.subtitle_download_in_progress = True video.save() self.stdout.write("Downloading subtitles for video '%s'...\n" % video.youtube_id) try: download_subtitles(video.youtube_id, language) self.stdout.write("Download is complete!\n") video.subtitles_downloaded = True video.subtitle_download_in_progress = False video.flagged_for_subtitle_download = False video.save() except NoSubs as e: video.flagged_for_subtitle_download = False video.subtitle_download_in_progress = False video.subtitles_downloaded = True video.save() self.stderr.write("No subtitles available\n") except Exception as e: self.stderr.write("Error in downloading subtitles: %s\n" % e) video.subtitle_download_in_progress = False video.flagged_for_subtitle_download = False video.save() force_job("subtitledownload", "Download Subtitles") return
def update(request): call_command("videoscan") # Could potentially be very slow, blocking request. force_job("videodownload", "Download Videos") force_job("subtitledownload", "Download Subtitles") default_language = Settings.get("subtitle_language") or "en" device = Device.get_own_device() zone = device.get_zone() context = { "default_language": default_language, "registered": Settings.get("registered"), "zone_id": zone.id if zone else None, "device_id": device.id, "video_count": VideoFile.objects.filter(percent_complete=100).count(), } return context
def update(request): call_command( "videoscan") # Could potentially be very slow, blocking request. force_job("videodownload", "Download Videos") force_job("subtitledownload", "Download Subtitles") default_language = Settings.get("subtitle_language") or "en" device = Device.get_own_device() zone = device.get_zone() context = { "default_language": default_language, "registered": Settings.get("registered"), "zone_id": zone.id if zone else None, "device_id": device.id, "video_count": VideoFile.objects.filter(percent_complete=100).count(), } return context
def start_subtitle_download(request): update_set = simplejson.loads(request.raw_post_data or "{}").get("update_set", "existing") language = simplejson.loads(request.raw_post_data or "{}").get("language", "") language_list = topicdata.LANGUAGE_LIST language_lookup = topicdata.LANGUAGE_LOOKUP # Reset the language current_language = Settings.get("subtitle_language") if language in language_list: Settings.set("subtitle_language", language) else: return JsonResponse({"error": "This language is not currently supported - please update the language list"}, status=500) language_name = language_lookup.get(language) # Get the json file with all srts request_url = "http://%s/static/data/subtitles/languages/%s_available_srts.json" % (settings.CENTRAL_SERVER_HOST, language) try: r = requests.get(request_url) r.raise_for_status() # will return none if 200, otherwise will raise HTTP error available_srts = set((r.json)["srt_files"]) except ConnectionError: return JsonResponse({"error": "The central server is currently offline."}, status=500) except HTTPError: return JsonResponse({"error": "No subtitles available on central server for %s (language code: %s); aborting." % (language_name, language)}, status=500) if update_set == "existing": videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), subtitles_downloaded=False, youtube_id__in=available_srts) else: videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), youtube_id__in=available_srts) if not videofiles: return JsonResponse({"info": "There aren't any subtitles available in %s (language code: %s) for your current videos." % (language_name, language)}, status=200) else: for videofile in videofiles: videofile.cancel_download = False if videofile.subtitle_download_in_progress: continue videofile.flagged_for_subtitle_download = True if update_set == "all": videofile.subtitles_downloaded = False videofile.save() force_job("subtitledownload", "Download Subtitles") return JsonResponse({})
def handle(self, *args, **options): language = Settings.get("subtitle_language") while True: # loop until the method is aborted if VideoFile.objects.filter(subtitle_download_in_progress=True).count() > 4: self.stderr.write("Maximum downloads are in progress; aborting.\n") return videos = VideoFile.objects.filter(flagged_for_subtitle_download=True, subtitle_download_in_progress=False) if videos.count() == 0: self.stdout.write("Nothing to download; aborting.\n") return video = videos[0] video.subtitle_download_in_progress = True video.save() self.stdout.write("Downloading subtitles for video '%s'... " % video.youtube_id) self.stdout.flush() try: download_subtitles(video.youtube_id, language) self.stdout.write("Download is complete!\n") video.subtitles_downloaded = True video.subtitle_download_in_progress = False video.flagged_for_subtitle_download = False video.save() except NoSubs as e: video.flagged_for_subtitle_download = False video.subtitle_download_in_progress = False video.subtitles_downloaded = True self.stdout.write("\n"); video.save() self.stderr.write("No subtitles available\n") except Exception as e: self.stderr.write("Error in downloading subtitles: %s\n" % e) video.subtitle_download_in_progress = False video.flagged_for_subtitle_download = False video.save() force_job("subtitledownload", "Download Subtitles") return
def start_subtitle_download(request): new_only = simplejson.loads(request.raw_post_data or "{}").get("new_only", False) language = simplejson.loads(request.raw_post_data or "{}").get("language", "") language_list = topicdata.LANGUAGE_LIST current_language = Settings.get("subtitle_language") new_only = new_only and (current_language == language) if language in language_list: Settings.set("subtitle_language", language) else: return JsonResponse({"error": "This language is not currently supported - please update the language list"}, status=500) if new_only: videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True), subtitles_downloaded=False) else: videofiles = VideoFile.objects.filter(Q(percent_complete=100) | Q(flagged_for_download=True)) for videofile in videofiles: videofile.cancel_download = False if videofile.subtitle_download_in_progress: continue videofile.flagged_for_subtitle_download = True if not new_only: videofile.subtitles_downloaded = False videofile.save() force_job("subtitledownload", "Download Subtitles") return JsonResponse({})
def set_as_registered(): force_job("syncmodels", "Secure Sync", "HOURLY") # now launches asynchronously Settings.set("registered", True)
def retry_video_download(request): """Clear any video still accidentally marked as in-progress, and restart the download job. """ VideoFile.objects.filter(download_in_progress=True).update(download_in_progress=False, percent_complete=0) force_job("videodownload", "Download Videos") return JsonResponse({})
def set_as_registered(): force_job("syncmodels", "Secure Sync", "HOURLY") Settings.set("registered", True)
def cancel_downloads(request): videofiles = VideoFile.objects.filter(Q(flagged_for_download=True) | Q(flagged_for_subtitle_download=True)) videofiles.update(cancel_download=True, flagged_for_download=False, flagged_for_subtitle_download=False) force_job("videodownload", stop=True) force_job("subtitledownload", stop=True) return JsonResponse({})