def updateVideoAndPlaylistReadableNames(self): # Makes sure every video and playlist has a unique "name" that can be used in URLs query = Video.all() all_videos = query.fetch(100000) for video in all_videos: potential_id = re.sub('[^a-z0-9]', '-', video.title.lower()) potential_id = re.sub( '-+$', '', potential_id) # remove any trailing dashes (see issue 1140) potential_id = re.sub( '^-+', '', potential_id) # remove any leading dashes (see issue 1526) if video.readable_id == potential_id: # id is unchanged continue number_to_add = 0 current_id = potential_id while True: query = Video.all() query.filter('readable_id=', current_id) if (query.get() is None): #id is unique so use it and break out video.readable_id = current_id video.put() break else: # id is not unique so will have to go through loop again number_to_add += 1 current_id = potential_id + '-' + number_to_add
def update(self, feedback): orig_video = feedback.video() if orig_video == None or type(orig_video).__name__ != "Video": return False readable_id = orig_video.readable_id query = Video.all() query.filter('readable_id =', readable_id) # The database currently contains multiple Video objects for a particular # video. Some are old. Some are due to a YouTube sync where the youtube urls # changed and our code was producing youtube_ids that ended with '_player'. # This hack gets the most recent valid Video object. key_id = 0 for v in query: if v.key().id() > key_id and not v.youtube_id.endswith('_player'): video = v key_id = v.key().id() # End of hack if video is not None and video.key() != orig_video.key(): logging.info("Retargeting Feedback %s from Video %s to Video %s", feedback.key().id(), orig_video.key().id(), video.key().id()) feedback.targets[0] = video.key() return True else: return False
def video_title_dicts(): live_video_dict = {} for video_playlist in VideoPlaylist.all().filter('live_association = ', True): live_video_dict[VideoPlaylist.video.get_value_for_datastore(video_playlist)] = True live_videos = filter(lambda video: video.key() in live_video_dict, Video.all()) return map(lambda video: {"title": video.title, "url": "/video/%s" % video.readable_id}, live_videos)
def update_video(self, video_youtube_id): v = Video.all().filter('youtube_id =', video_youtube_id).get() if v is not None: if v.key() not in self.videos: self.videos.append(v.key()) return True else: logging.debug("Youtube ID %s not in datastore" % video_youtube_id)
def test_derive_key_name_from_video(self): self._set_responses_xrange(BATCH_SIZE) _task_handler('UUID') videos = Video.all().fetch(BATCH_SIZE) for v in videos: key = VideoSubtitles.get_key_name('en', v.youtube_id) subs = VideoSubtitles.get_by_key_name(key) self.assertIsNotNone(subs)
def updateVideoAndPlaylistReadableNames(self): # Makes sure every video and playlist has a unique "name" that can be used in URLs query = Video.all() all_videos = query.fetch(100000) for video in all_videos: potential_id = re.sub('[^a-z0-9]', '-', video.title.lower()); potential_id = re.sub('-+$', '', potential_id) # remove any trailing dashes (see issue 1140) potential_id = re.sub('^-+', '', potential_id) # remove any leading dashes (see issue 1526) if video.readable_id == potential_id: # id is unchanged continue number_to_add = 0 current_id = potential_id while True: query = Video.all() query.filter('readable_id=', current_id) if (query.get() is None): #id is unique so use it and break out video.readable_id = current_id video.put() break else: # id is not unique so will have to go through loop again number_to_add+=1 current_id = potential_id+'-'+number_to_add
def get(self): user_data = UserData.current() video_points_total = 0 if user_data: video = None key_str = self.request_string("video_key") if key_str: key = db.Key(key_str) app_id = os.environ['APPLICATION_ID'] if key.app() != app_id: new_key = db.Key.from_path(key.kind(), key.id() or key.name(), _app=app_id) logging.warning( "Key '%s' had invalid app_id '%s'. Changed to new key '%s'", str(key), key.app(), str(new_key)) key = new_key video = db.get(key) else: youtube_id = self.request_string("youtube_id") if youtube_id: video = Video.all().filter('youtube_id =', youtube_id).get() if video: # Seconds watched is restricted by both the scrubber's position # and the amount of time spent on the video page # so we know how *much* of each video each student has watched seconds_watched = int( self.request_float("seconds_watched", default=0)) last_second_watched = int( self.request_float("last_second_watched", default=0)) user_video, video_log, video_points_total = VideoLog.add_entry( user_data, video, seconds_watched, last_second_watched) user_points_html = self.render_jinja2_template_to_string( "user_points_only.html", user_points(user_data)) json = simplejson.dumps( { "user_points_html": user_points_html, "video_points": video_points_total }, ensure_ascii=False) self.response.out.write(json)
def video_title_dicts(): live_video_dict = {} for video_playlist in VideoPlaylist.all().filter('live_association = ', True): live_video_dict[VideoPlaylist.video.get_value_for_datastore( video_playlist)] = True live_videos = filter(lambda video: video.key() in live_video_dict, Video.all()) return map( lambda video: { "title": video.title, "url": "/video/%s" % video.readable_id }, live_videos)
def get_playlist_videos_json(playlist_title): query = Playlist.all() query.filter('title =', playlist_title) playlist = query.get() video_query = Video.all() video_query.filter('playlists = ', playlist_title) video_key_dict = Video.get_dict(video_query, lambda video: video.key()) video_playlist_query = VideoPlaylist.all() video_playlist_query.filter('playlist =', playlist) video_playlist_query.filter('live_association =', True) video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query) return json.dumps(get_playlist_video_api_dicts(playlist, video_key_dict, video_playlist_key_dict), indent=4)
def get_video_library_json_compressed(): playlist_api_dicts = [] playlists = get_all_topic_playlists() video_key_dict = Video.get_dict(Video.all(), lambda video: video.key()) video_playlist_query = VideoPlaylist.all() video_playlist_query.filter('live_association =', True) video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query) for playlist in playlists: playlist_api_dict = JsonApiDict.playlist(playlist) playlist_api_dict["videos"] = get_playlist_video_api_dicts(playlist, video_key_dict, video_playlist_key_dict) playlist_api_dicts.append(playlist_api_dict) # We compress this huge json payload so it'll fit in memcache return zlib.compress(json.dumps(playlist_api_dicts, indent=4))
def test_process_next_batch_on_nonempty_cursor(self): offset = 3 # these should be skipped, they'll DownloadError for i in xrange(0, offset): Video(youtube_id=str(i)).put() # these should be downloaded self._set_responses_xrange(offset, BATCH_SIZE + offset) query = Video.all() query.fetch(offset) cursor = query.cursor() _task_handler('UUID', cursor=cursor) self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def get_video_library_json_compressed(): playlist_api_dicts = [] playlists = get_all_topic_playlists() video_key_dict = Video.get_dict(Video.all(), lambda video: video.key()) video_playlist_query = VideoPlaylist.all() video_playlist_query.filter('live_association =', True) video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query) for playlist in playlists: playlist_api_dict = JsonApiDict.playlist(playlist) playlist_api_dict["videos"] = get_playlist_video_api_dicts( playlist, video_key_dict, video_playlist_key_dict) playlist_api_dicts.append(playlist_api_dict) # We compress this huge json payload so it'll fit in memcache return zlib.compress(json.dumps(playlist_api_dicts, indent=4))
def youtube_get_video_data_dict(youtube_id): yt_service = third_party.gdata.youtube.service.YouTubeService() # Now that we run these queries from the App Engine servers, we need to # explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and # throttled by YouTube's "Too many request" quota yt_service.developer_key = "AI39si5NKByjOThtM6t1gnmg4N9HkzGPHHkVvRUybKuPG53277wY0Bs4zcvi-G4JiFioRs0738gaE01k1rX-_6-mIHp9jg1twQ" yt_service.client_id = "n/a" logging.info("trying to get info for youtube_id: %s" % youtube_id) try: video = yt_service.GetYouTubeVideoEntry(video_id=youtube_id) except: video = None if video: video_data = {"youtube_id" : youtube_id, "title" : video.media.title.text.decode('utf-8'), "url" : video.media.player.url.decode('utf-8'), "duration" : int(video.media.duration.seconds)} if video.statistics: video_data["views"] = int(video.statistics.view_count) video_data["description"] = (video.media.description.text or '').decode('utf-8') video_data["keywords"] = (video.media.keywords.text or '').decode('utf-8') potential_id = re.sub('[^a-z0-9]', '-', video_data["title"].lower()); potential_id = re.sub('-+$', '', potential_id) # remove any trailing dashes (see issue 1140) potential_id = re.sub('^-+', '', potential_id) # remove any leading dashes (see issue 1526) number_to_add = 0 current_id = potential_id while True: query = Video.all() query.filter('readable_id=', current_id) if (query.get() is None): #id is unique so use it and break out video_data["readable_id"] = current_id break else: # id is not unique so will have to go through loop again number_to_add+=1 current_id = potential_id+'-'+number_to_add return video_data return None
def get(self): user_data = UserData.current() video_points_total = 0 if user_data: video = None key_str = self.request_string("video_key") if key_str: key = db.Key(key_str) app_id = os.environ['APPLICATION_ID'] if key.app() != app_id: new_key = db.Key.from_path( key.kind(), key.id() or key.name(), _app=app_id) logging.warning("Key '%s' had invalid app_id '%s'. Changed to new key '%s'", str(key), key.app(), str(new_key)) key = new_key video = db.get(key) else: youtube_id = self.request_string("youtube_id") if youtube_id: video = Video.all().filter('youtube_id =', youtube_id).get() if video: # Seconds watched is restricted by both the scrubber's position # and the amount of time spent on the video page # so we know how *much* of each video each student has watched seconds_watched = int(self.request_float("seconds_watched", default=0)) last_second_watched = int(self.request_float("last_second_watched", default=0)) user_video, video_log, video_points_total = VideoLog.add_entry(user_data, video, seconds_watched, last_second_watched) user_points_html = self.render_jinja2_template_to_string("user_points_only.html", user_points(user_data)) json = simplejson.dumps({"user_points_html": user_points_html, "video_points": video_points_total}, ensure_ascii=False) self.response.out.write(json)
def indexVideoData(self): videos = Video.all().fetch(10000) for video in videos: video.index() video.indexed_title_changed()
def api_get_videos(): videos = Video.all() api_response = jsonify(ok=True, result=videos) return api_response
def updateVideoAndPlaylistData(self): yt_service = gdata.youtube.service.YouTubeService() # Now that we run these queries from the App Engine servers, we need to # explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and # throttled by YouTube's "Too many request" quota yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ" yt_service.client_id = "n/a" video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id) video_playlist_key_dict = VideoPlaylist.get_key_dict( VideoPlaylist.all()) association_generation = int( Setting.last_youtube_sync_generation_start()) logging.info("Fetching playlists") playlist_start_index = 1 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri= 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index) while len(playlist_feed.entry) > 0: for playlist in playlist_feed.entry: logging.info("Playlist: %s", playlist.id.text) playlist_id = playlist.id.text.replace( 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/', '') playlist_uri = playlist.id.text.replace( 'users/KhanAcademyHebrew/', '') query = Playlist.all() query.filter('youtube_id =', playlist_id) playlist_data = query.get() if not playlist_data: playlist_data = Playlist(youtube_id=playlist_id) logging.info('Creating Playlist: %s', playlist.title.text) playlist_data.url = playlist_uri playlist_data.title = playlist.title.text.decode("utf-8") playlist_data.description = playlist.description.text.decode( "utf-8") playlist_data.tags = [] for category in playlist.category: if "tags.cat" in category.scheme: playlist_data.tags.append(category.term) playlist_data.put() for i in range(0, 10): start_index = i * 50 + 1 video_feed = yt_service.GetYouTubePlaylistVideoFeed( uri=playlist_uri + '?start-index=' + str(start_index) + '&max-results=50') video_data_list = [] if len(video_feed.entry) <= 0: # No more videos in playlist break for video in video_feed.entry: if not video.media.player: logging.warning( "Could not parse video - skipping... (%s, %s)", video, video.media) continue video_id = cgi.parse_qs( urlparse( video.media.player.url).query)['v'][0].decode( 'utf-8') video_data = None if video_youtube_id_dict.has_key(video_id): video_data = video_youtube_id_dict[video_id] logging.info( 'Found Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) if not video_data: video_data = Video(youtube_id=video_id) logging.info( 'Creating Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) video_data.title = video.media.title.text.decode( 'utf-8') video_data.url = video.media.player.url.decode('utf-8') video_data.duration = int(video.media.duration.seconds) if video.statistics: video_data.views = int(video.statistics.view_count) if video.media.description.text is not None: video_data.description = video.media.description.text.decode( 'utf-8') else: video_data.decription = ' ' if video.media.keywords.text: video_data.keywords = video.media.keywords.text.decode( 'utf-8') else: video_data.keywords = '' video_data.position = video.position video_data_list.append(video_data) db.put(video_data_list) playlist_videos = [] for video_data in video_data_list: playlist_video = None if video_playlist_key_dict.has_key( playlist_data.key()): if video_playlist_key_dict[ playlist_data.key()].has_key( video_data.key()): playlist_video = video_playlist_key_dict[ playlist_data.key()][video_data.key()] if not playlist_video: playlist_video = VideoPlaylist( playlist=playlist_data.key(), video=video_data.key()) logging.info('Creating VideoPlaylist: %s, %s', playlist_data.title, video_data.title) else: logging.info('Updating VideoPlaylist: %s, %s', playlist_video.playlist.title, playlist_video.video.title) playlist_video.last_live_association_generation = association_generation playlist_video.video_position = int( video_data.position.text) playlist_videos.append(playlist_video) db.put(playlist_videos) # Check next set of playlists playlist_start_index += 50 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri= 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
def get(self, readable_id=""): # This method displays a video in the context of a particular topic. # To do that we first need to find the appropriate topic. If we aren't # given the topic title in a query param, we need to find a topic that # the video is a part of. That requires finding the video, given it readable_id # or, to support old URLs, it's youtube_id. video = None topic = None video_id = self.request.get('v') topic_id = self.request_string('topic', default="") readable_id = urllib.unquote(readable_id).decode("utf-8") readable_id = re.sub( '-+$', '', readable_id) # remove any trailing dashes (see issue 1140) # If either the readable_id or topic title is missing, # redirect to the canonical URL that contains them redirect_to_canonical_url = False if video_id: # Support for old links query = Video.all() query.filter('youtube_id =', video_id) video = query.get() if not video: raise MissingVideoException( "Missing video w/ youtube id '%s'" % video_id) readable_id = video.readable_id topic = video.first_topic() if not topic: raise MissingVideoException( "No topic has video w/ youtube id '%s'" % video_id) redirect_to_canonical_url = True if topic_id is not None and len(topic_id) > 0: topic = Topic.get_by_id(topic_id) key_id = 0 if not topic else topic.key().id() # If a topic_id wasn't specified or the specified topic wasn't found # use the first topic for the requested video. if topic is None: # Get video by readable_id just to get the first topic for the video video = Video.get_for_readable_id(readable_id) if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) topic = video.first_topic() if not topic: raise MissingVideoException("No topic has video '%s'" % readable_id) redirect_to_canonical_url = True exid = self.request_string('exid', default=None) if redirect_to_canonical_url: qs = {'topic': topic.id} if exid: qs['exid'] = exid urlpath = "/video/%s" % urllib.quote(readable_id) url = urlparse.urlunparse( ('', '', urlpath, '', urllib.urlencode(qs), '')) self.redirect(url, True) return # If we got here, we have a readable_id and a topic, so we can display # the topic and the video in it that has the readable_id. Note that we don't # query the Video entities for one with the requested readable_id because in some # cases there are multiple Video objects in the datastore with the same readable_id # (e.g. there are 2 "Order of Operations" videos). videos = Topic.get_cached_videos_for_topic(topic) previous_video = None next_video = None for v in videos: if v.readable_id == readable_id: v.selected = 'selected' video = v elif video is None: previous_video = v else: next_video = v break # If we're at the beginning or end of a topic, show the adjacent topic. # previous_topic/next_topic are the topic to display. # previous_video_topic/next_video_topic are the subtopics the videos # are actually in. previous_topic = None previous_video_topic = None next_topic = None next_video_topic = None if not previous_video: previous_topic = topic while not previous_video: previous_topic = previous_topic.get_previous_topic() if previous_topic: (previous_video, previous_video_topic ) = previous_topic.get_last_video_and_topic() else: break if not next_video: next_topic = topic while not next_video: next_topic = next_topic.get_next_topic() if next_topic: (next_video, next_video_topic ) = next_topic.get_first_video_and_topic() else: break if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) if App.offline_mode: video_path = "/videos/" + get_mangled_topic_name( topic.id) + "/" + video.readable_id + ".flv" else: video_path = video.download_video_url() if video.description == video.title: video.description = None related_exercises = video.related_exercises() button_top_exercise = None if related_exercises: def ex_to_dict(exercise): return { 'name': exercise.display_name, 'url': exercise.relative_url, } button_top_exercise = ex_to_dict(related_exercises[0]) user_video = UserVideo.get_for_video_and_user_data( video, UserData.current(), insert_if_missing=True) awarded_points = 0 if user_video: awarded_points = user_video.points subtitles_key_name = VideoSubtitles.get_key_name( 'en', video.youtube_id) subtitles = VideoSubtitles.get_by_key_name(subtitles_key_name) subtitles_json = None if subtitles: subtitles_json = subtitles.load_json() template_values = { 'topic': topic, 'video': video, 'videos': videos, 'video_path': video_path, 'video_points_base': consts.VIDEO_POINTS_BASE, 'subtitles_json': subtitles_json, 'button_top_exercise': button_top_exercise, 'related_exercises': [], # disabled for now 'previous_topic': previous_topic, 'previous_video': previous_video, 'previous_video_topic': previous_video_topic, 'next_topic': next_topic, 'next_video': next_video, 'next_video_topic': next_video_topic, 'selected_nav_link': 'watch', 'awarded_points': awarded_points, 'issue_labels': ('Component-Videos,Video-%s' % readable_id), 'author_profile': 'https://plus.google.com/103970106103092409324' } template_values = qa.add_template_values(template_values, self.request) bingo([ 'struggling_videos_landing', 'suggested_activity_videos_landing', 'suggested_activity_videos_landing_binary', ]) self.render_jinja2_template('viewvideo.html', template_values)
def get(self): from exercises import attempt_problem login_user = UserData.current() exercises_list = [exercise for exercise in Exercise.all()] videos_list = [video for video in Video.all()] user_count = self.request_int('users', 5) for user_id in xrange(0, user_count): # Create a new user first_name = random.choice(CreateRandomGoalData.first_names) last_name = random.choice(CreateRandomGoalData.last_names) nickname = "%s %s" % (first_name, last_name) email = 'test_%i@automatedrandomdata' % user_id user = users.User(email) logging.info("Creating user %s: (%i/%i)" % (nickname, user_id + 1, user_count)) user_data = UserData.get_or_insert( key_name="test_user_%i" % user_id, user=user, current_user=user, user_id=str(user_id), moderator=False, last_login=datetime.now(), proficient_exercises=[], suggested_exercises=[], need_to_reassess=True, points=0, coaches=[login_user.user_email], user_email=email, user_nickname=nickname, ) user_data.put() # Delete user exercise & video progress query = UserExercise.all() query.filter('user = '******'user = '******'type': 'GoalObjectiveExerciseProficiency', 'exercise': random.choice(exercises_list)}) for objective in xrange(1, random.randint(2, 4)): obj_descriptors.append({ 'type': 'GoalObjectiveWatchVideo', 'video': random.choice(videos_list)}) title = first_name + "'s Goal #" + str(goal_idx) logging.info("Creating goal " + title) objectives = GoalObjective.from_descriptors(obj_descriptors, user_data) goal = Goal(parent=user_data, title=title, objectives=objectives) user_data.save_goal(goal) for objective in obj_descriptors: if objective['type'] == 'GoalObjectiveExerciseProficiency': user_exercise = user_data.get_or_insert_exercise( objective['exercise']) chooser = random.randint(1, 120) if chooser >= 60: continue elif chooser > 15: count = 1 hints = 0 elif chooser < 7: count = 20 hints = 0 else: count = 25 hints = 1 logging.info( "Starting exercise: %s (%i problems, %i hints)" % (objective['exercise'].name, count, hints * count)) for i in xrange(1, count): attempt_problem(user_data, user_exercise, i, 1, 'TEST', 'TEST', 'TEST', True, hints, 0, "TEST", 'TEST', '0.0.0.0') elif objective['type'] == 'GoalObjectiveWatchVideo': seconds = random.randint(1, 1200) logging.info("Watching %i seconds of video %s" % (seconds, objective['video'].title)) VideoLog.add_entry(user_data, objective['video'], seconds, 0, detect_cheat=False) self.response.out.write('OK')
def get(self): self.response.out.write('<html>') videos = Video.all() for video in videos: self.response.out.write('<P>Title: ' + video.title)
def process_search(): search_query = request.GET.get("search_query", "").strip() query = search_query.lower() show_daverank = False results = False number_pages = 10 number_videos = 5 # Move this stuff to its own procedure tomorrow! if query.find("--") == 0: if query.find("--forum") == 0: redirect_url = "http://www.udacity-forums.com/cs101/search/?q=" + urllib.quote(query[8:]) return redirect(redirect_url) if query.find("--cs373") == 0: redirect_url = "http://www.udacity-forums.com/cs373/search/?q=" + urllib.quote(query[8:]) return redirect(redirect_url) if query.find("--python") == 0: redirect_url = "http://docs.python.org/search.html?q=" + urllib.quote(query[9:]) return redirect(redirect_url) if query.find("--searchwithpeterdotinfo") == 0: redirect_url = "http://searchwithpeter.info/secretplans.html?q=" + urllib.quote(query[25:]) return redirect(redirect_url) if query.find("--showmore") == 0: query = query[11:] search_query = query number_pages = 20 number_videos = 10 if query.find("--daverank") == 0: query = query[11:] search_query = query show_daverank = True if query.find("python") == 0: pyquery = query[7:] else: pyquery = query ddgurl_root = "http://duckduckgo.com/?q=python+" ddgurl_suffix = urllib.quote(pyquery) + "&format=json" response = urllib.urlopen(ddgurl_root + ddgurl_suffix) response_json = response.read() pythonterm = json.loads(response_json) if pythonterm: pyterm_info = {} if pythonterm["AbstractSource"] == "Python Documentation": pyterm = BeautifulSoup(pythonterm["AbstractText"]) try: pyterm_code = pyterm.find("code").string pyterm.pre.decompose() pyterm_info["code"] = pyterm_code except: pyterm_info["code"] = None pyterm_desc = pyterm.get_text() pyterm_info["desc"] = pyterm_desc pyterm_info["url"] = pythonterm["AbstractURL"] results = True else: pyterm_info = None query_words = query.split() for word in query_words: if word in stopwords: query_words.remove(word) query_urls = [] for term in query_words: # Get all SearchTerm objects that match the search_query. q = SearchTerm.all().filter("term =", term).get() if q: query_urls.append(set(q.urls)) if query_urls: query_url_set = set.intersection(*query_urls) query_url_list = list(query_url_set) if len(query_url_list) > 0: results = True if len(query_url_list) > 30: query_url_list = query_url_list[0:30] page_results = Page.all().filter("url IN", query_url_list).order("-dave_rank").fetch(number_pages) page_dicts = [] for page in page_results: page_info = {} query_index = page.text.find(query) if query_index != -1: i = page.text.find(" ", query_index - 25) excerpt_words = page.text[i:].split(" ") page_info["exact_match"] = True else: excerpt_words = page.text.split(" ") page_info["exact_match"] = False excerpt = " ".join(excerpt_words[:50]) page_info["text"] = excerpt page_info["title"] = page.title page_info["url"] = page.url page_info["daverank"] = page.dave_rank page_info["doc"] = page.doc page_dicts.append(page_info) page_dicts.sort(key=itemgetter("exact_match"), reverse=True) video_results = Video.all().filter("url IN", query_url_list).order("-views").fetch(number_videos) video_dicts = [] for video in video_results: video_info = {} subtitles = video.text.lower() query_index = subtitles.find(query) time_string = "" if query_index != -1: subtitle_list = subtitles.splitlines() for phrase in subtitle_list: if phrase.find(query) != -1: timestamp_index = subtitle_list.index(phrase) - 1 timestamp = subtitle_list[timestamp_index] if len(timestamp) > 1: minutes = timestamp[3:5] seconds = timestamp[6:8] time_string = "#t=" + minutes + "m" + seconds + "s" start = 60 * int(minutes) + int(seconds) if time_string: url = video.url + time_string video_info["exact_match"] = True else: url = video.url start = 0 video_info["exact_match"] = False video_info["title"] = video.title video_info["url"] = url video_info["subtitle"] = video.text[-20:query_index:20] video_info["id"] = video.id video_info["start"] = start video_dicts.append(video_info) video_dicts.sort(key=itemgetter("exact_match"), reverse=True) else: page_dicts = None video_dicts = None query_string_words = query.split() return template( "templates/results", search_query=search_query, query_string_words=query_string_words, page_dicts=page_dicts, video_dicts=video_dicts, pyterm_info=pyterm_info, show_daverank=show_daverank, results=results, )
def get(self): videos = Video.all() render_template(self, 'index.html', {'videos': videos})
def get(self): videos = Video.all() render_template(self, 'index.html', { 'videos': videos })
def get_video(self, youtube_id): return Video.all().filter('youtube_id =', youtube_id).get()
def library_content_html(): # No cache found -- regenerate HTML smart_history = getSmartHistoryContent() all_playlists = [] dict_videos = {} dict_videos_counted = {} dict_playlists = {} dict_playlists_by_title = {} dict_video_playlists = {} async_queries = [ Video.all(), Playlist.all(), VideoPlaylist.all().filter('live_association = ', True).order('video_position'), ] results = util.async_queries(async_queries) for video in results[0].get_result(): dict_videos[video.key()] = video for playlist in results[1].get_result(): dict_playlists[playlist.key()] = playlist if playlist.title in topics_list: dict_playlists_by_title[playlist.title] = playlist for video_playlist in results[2].get_result(): playlist_key = VideoPlaylist.playlist.get_value_for_datastore( video_playlist) video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist) if dict_videos.has_key(video_key) and dict_playlists.has_key( playlist_key): video = dict_videos[video_key] playlist = dict_playlists[playlist_key] fast_video_playlist_dict = {"video": video, "playlist": playlist} if dict_video_playlists.has_key(playlist_key): dict_video_playlists[playlist_key].append( fast_video_playlist_dict) else: dict_video_playlists[playlist_key] = [fast_video_playlist_dict] if dict_playlists_by_title.has_key(playlist.title): # Only count videos in topics_list dict_videos_counted[video.youtube_id] = True # Update count of all distinct videos associated w/ a live playlist Setting.count_videos(len(dict_videos_counted.keys())) for topic in topics_list: if topic in dict_playlists_by_title: playlist = dict_playlists_by_title[topic] playlist_key = playlist.key() playlist_videos = dict_video_playlists.get(playlist_key) or [] if not playlist_videos: logging.error('Playlist %s has no videos!', playlist.title) playlist_data = { 'title': topic, 'topic': topic, 'playlist': playlist, 'videos': playlist_videos, 'next': None } all_playlists.append(playlist_data) playlist_data_prev = None for playlist_data in all_playlists: if playlist_data_prev: playlist_data_prev['next'] = playlist_data playlist_data_prev = playlist_data # Separating out the columns because the formatting is a little different on each column template_values = { 'App': App, 'all_playlists': all_playlists, 'smart_history': smart_history, } html = shared_jinja.get().render_template("library_content_template.html", **template_values) # Set shared date of last generated content Setting.cached_library_content_date(str(datetime.datetime.now())) return html
def process_search(): search_query = request.GET.get('search_query', '').strip() query = search_query.lower() show_daverank = False results = False number_pages = 10 number_videos = 5 #Move this stuff to its own procedure tomorrow! if query.find('--') == 0: if query.find('--forum') == 0: redirect_url = 'http://www.udacity-forums.com/cs101/search/?q=' + urllib.quote( query[8:]) return redirect(redirect_url) if query.find('--cs373') == 0: redirect_url = 'http://www.udacity-forums.com/cs373/search/?q=' + urllib.quote( query[8:]) return redirect(redirect_url) if query.find('--python') == 0: redirect_url = 'http://docs.python.org/search.html?q=' + urllib.quote( query[9:]) return redirect(redirect_url) if query.find('--searchwithpeterdotinfo') == 0: redirect_url = 'http://searchwithpeter.info/secretplans.html?q=' + urllib.quote( query[25:]) return redirect(redirect_url) if query.find('--showmore') == 0: query = query[11:] search_query = query number_pages = 20 number_videos = 10 if query.find('--daverank') == 0: query = query[11:] search_query = query show_daverank = True if query.find('python') == 0: pyquery = query[7:] else: pyquery = query ddgurl_root = 'http://duckduckgo.com/?q=python+' ddgurl_suffix = urllib.quote(pyquery) + '&format=json' response = urllib.urlopen(ddgurl_root + ddgurl_suffix) response_json = response.read() pythonterm = json.loads(response_json) if pythonterm: pyterm_info = {} if pythonterm['AbstractSource'] == 'Python Documentation': pyterm = BeautifulSoup(pythonterm['AbstractText']) try: pyterm_code = pyterm.find('code').string pyterm.pre.decompose() pyterm_info['code'] = pyterm_code except: pyterm_info['code'] = None pyterm_desc = pyterm.get_text() pyterm_info['desc'] = pyterm_desc pyterm_info['url'] = pythonterm['AbstractURL'] results = True else: pyterm_info = None query_words = query.split() for word in query_words: if word in stopwords: query_words.remove(word) query_urls = [] for term in query_words: # Get all SearchTerm objects that match the search_query. q = SearchTerm.all().filter('term =', term).get() if q: query_urls.append(set(q.urls)) if query_urls: query_url_set = set.intersection(*query_urls) query_url_list = list(query_url_set) if len(query_url_list) > 0: results = True if len(query_url_list) > 30: query_url_list = query_url_list[0:30] page_results = Page.all().filter( 'url IN', query_url_list).order('-dave_rank').fetch(number_pages) page_dicts = [] for page in page_results: page_info = {} query_index = page.text.find(query) if query_index != -1: i = page.text.find(' ', query_index - 25) excerpt_words = page.text[i:].split(' ') page_info['exact_match'] = True else: excerpt_words = page.text.split(' ') page_info['exact_match'] = False excerpt = ' '.join(excerpt_words[:50]) page_info['text'] = excerpt page_info['title'] = page.title page_info['url'] = page.url page_info['daverank'] = page.dave_rank page_info['doc'] = page.doc page_dicts.append(page_info) page_dicts.sort(key=itemgetter('exact_match'), reverse=True) video_results = Video.all().filter( 'url IN', query_url_list).order('-views').fetch(number_videos) video_dicts = [] for video in video_results: video_info = {} subtitles = video.text.lower() query_index = subtitles.find(query) time_string = '' if query_index != -1: subtitle_list = subtitles.splitlines() for phrase in subtitle_list: if phrase.find(query) != -1: timestamp_index = subtitle_list.index(phrase) - 1 timestamp = subtitle_list[timestamp_index] if len(timestamp) > 1: minutes = timestamp[3:5] seconds = timestamp[6:8] time_string = '#t=' + minutes + 'm' + seconds + 's' start = 60 * int(minutes) + int(seconds) if time_string: url = video.url + time_string video_info['exact_match'] = True else: url = video.url start = 0 video_info['exact_match'] = False video_info['title'] = video.title video_info['url'] = url video_info['subtitle'] = video.text[-20:query_index:20] video_info['id'] = video.id video_info['start'] = start video_dicts.append(video_info) video_dicts.sort(key=itemgetter('exact_match'), reverse=True) else: page_dicts = None video_dicts = None query_string_words = query.split() return template('templates/results', search_query=search_query, query_string_words=query_string_words, page_dicts=page_dicts, video_dicts=video_dicts, pyterm_info=pyterm_info, show_daverank=show_daverank, results=results)
def get(self, readable_id=""): # This method displays a video in the context of a particular playlist. # To do that we first need to find the appropriate playlist. If we aren't # given the playlist title in a query param, we need to find a playlist that # the video is a part of. That requires finding the video, given it readable_id # or, to support old URLs, it's youtube_id. video = None playlist = None video_id = self.request.get('v') playlist_title = self.request_string('playlist', default="") or self.request_string('p', default="") readable_id = urllib.unquote(readable_id) readable_id = re.sub('-+$', '', readable_id) # remove any trailing dashes (see issue 1140) # If either the readable_id or playlist title is missing, # redirect to the canonical URL that contains them redirect_to_canonical_url = False if video_id: # Support for old links query = Video.all() query.filter('youtube_id =', video_id) video = query.get() if not video: raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id) readable_id = video.readable_id playlist = video.first_playlist() if not playlist: raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id) redirect_to_canonical_url = True if playlist_title is not None and len(playlist_title) > 0: query = Playlist.all().filter('title =', playlist_title) key_id = 0 for p in query: if p.key().id() > key_id and not p.youtube_id.endswith('_player'): playlist = p key_id = p.key().id() # If a playlist_title wasn't specified or the specified playlist wasn't found # use the first playlist for the requested video. if playlist is None: # Get video by readable_id just to get the first playlist for the video video = Video.get_for_readable_id(readable_id) if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) playlist = video.first_playlist() if not playlist: raise MissingVideoException("Missing video '%s'" % readable_id) redirect_to_canonical_url = True exid = self.request_string('exid', default=None) if redirect_to_canonical_url: qs = {'playlist': playlist.title} if exid: qs['exid'] = exid urlpath = "/video/%s" % urllib.quote(readable_id) url = urlparse.urlunparse(('', '', urlpath, '', urllib.urlencode(qs), '')) self.redirect(url, True) return # If we got here, we have a readable_id and a playlist_title, so we can display # the playlist and the video in it that has the readable_id. Note that we don't # query the Video entities for one with the requested readable_id because in some # cases there are multiple Video objects in the datastore with the same readable_id # (e.g. there are 2 "Order of Operations" videos). videos = VideoPlaylist.get_cached_videos_for_playlist(playlist) previous_video = None next_video = None for v in videos: if v.readable_id == readable_id: v.selected = 'selected' video = v elif video is None: previous_video = v elif next_video is None: next_video = v if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) if App.offline_mode: video_path = "/videos/" + get_mangled_playlist_name(playlist_title) + "/" + video.readable_id + ".flv" else: video_path = video.download_video_url() if video.description == video.title: video.description = None related_exercises = video.related_exercises() button_top_exercise = None if related_exercises: def ex_to_dict(exercise): return { 'name': exercise.display_name, 'url': exercise.relative_url, } button_top_exercise = ex_to_dict(related_exercises[0]) user_video = UserVideo.get_for_video_and_user_data(video, UserData.current(), insert_if_missing=True) awarded_points = 0 if user_video: awarded_points = user_video.points template_values = { 'playlist': playlist, 'video': video, 'videos': videos, 'video_path': video_path, 'video_points_base': consts.VIDEO_POINTS_BASE, 'button_top_exercise': button_top_exercise, 'related_exercises': [], # disabled for now 'previous_video': previous_video, 'next_video': next_video, 'selected_nav_link': 'watch', 'awarded_points': awarded_points, 'issue_labels': ('Component-Videos,Video-%s' % readable_id), 'author_profile': 'https://plus.google.com/103970106103092409324' } template_values = qa.add_template_values(template_values, self.request) bingo(['struggling_videos_landing', 'homepage_restructure_videos_landing']) self.render_jinja2_template('viewvideo.html', template_values)
def get(self, readable_id=""): # This method displays a video in the context of a particular topic. # To do that we first need to find the appropriate topic. If we aren't # given the topic title in a query param, we need to find a topic that # the video is a part of. That requires finding the video, given it readable_id # or, to support old URLs, it's youtube_id. video = None topic = None video_id = self.request.get('v') topic_id = self.request_string('topic', default="") readable_id = urllib.unquote(readable_id).decode("utf-8") readable_id = re.sub('-+$', '', readable_id) # remove any trailing dashes (see issue 1140) # If either the readable_id or topic title is missing, # redirect to the canonical URL that contains them redirect_to_canonical_url = False if video_id: # Support for old links query = Video.all() query.filter('youtube_id =', video_id) video = query.get() if not video: raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id) readable_id = video.readable_id topic = video.first_topic() if not topic: raise MissingVideoException("No topic has video w/ youtube id '%s'" % video_id) redirect_to_canonical_url = True if topic_id is not None and len(topic_id) > 0: topic = Topic.get_by_id(topic_id) key_id = 0 if not topic else topic.key().id() # If a topic_id wasn't specified or the specified topic wasn't found # use the first topic for the requested video. if topic is None: # Get video by readable_id just to get the first topic for the video video = Video.get_for_readable_id(readable_id) if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) topic = video.first_topic() if not topic: raise MissingVideoException("No topic has video '%s'" % readable_id) redirect_to_canonical_url = True exid = self.request_string('exid', default=None) if redirect_to_canonical_url: qs = {'topic': topic.id} if exid: qs['exid'] = exid urlpath = "/video/%s" % urllib.quote(readable_id) url = urlparse.urlunparse(('', '', urlpath, '', urllib.urlencode(qs), '')) self.redirect(url, True) return # If we got here, we have a readable_id and a topic, so we can display # the topic and the video in it that has the readable_id. Note that we don't # query the Video entities for one with the requested readable_id because in some # cases there are multiple Video objects in the datastore with the same readable_id # (e.g. there are 2 "Order of Operations" videos). videos = Topic.get_cached_videos_for_topic(topic) previous_video = None next_video = None for v in videos: if v.readable_id == readable_id: v.selected = 'selected' video = v elif video is None: previous_video = v else: next_video = v break # If we're at the beginning or end of a topic, show the adjacent topic. # previous_topic/next_topic are the topic to display. # previous_video_topic/next_video_topic are the subtopics the videos # are actually in. previous_topic = None previous_video_topic = None next_topic = None next_video_topic = None if not previous_video: previous_topic = topic while not previous_video: previous_topic = previous_topic.get_previous_topic() if previous_topic: (previous_video, previous_video_topic) = previous_topic.get_last_video_and_topic() else: break if not next_video: next_topic = topic while not next_video: next_topic = next_topic.get_next_topic() if next_topic: (next_video, next_video_topic) = next_topic.get_first_video_and_topic() else: break if video is None: raise MissingVideoException("Missing video '%s'" % readable_id) if App.offline_mode: video_path = "/videos/" + get_mangled_topic_name(topic.id) + "/" + video.readable_id + ".flv" else: video_path = video.download_video_url() if video.description == video.title: video.description = None related_exercises = video.related_exercises() button_top_exercise = None if related_exercises: def ex_to_dict(exercise): return { 'name': exercise.display_name, 'url': exercise.relative_url, } button_top_exercise = ex_to_dict(related_exercises[0]) user_video = UserVideo.get_for_video_and_user_data(video, UserData.current(), insert_if_missing=True) awarded_points = 0 if user_video: awarded_points = user_video.points template_values = { 'topic': topic, 'video': video, 'videos': videos, 'video_path': video_path, 'video_points_base': consts.VIDEO_POINTS_BASE, 'button_top_exercise': button_top_exercise, 'related_exercises': [], # disabled for now 'previous_topic': previous_topic, 'previous_video': previous_video, 'previous_video_topic': previous_video_topic, 'next_topic': next_topic, 'next_video': next_video, 'next_video_topic': next_video_topic, 'selected_nav_link': 'watch', 'awarded_points': awarded_points, 'issue_labels': ('Component-Videos,Video-%s' % readable_id), 'author_profile': 'https://plus.google.com/103970106103092409324', 'is_mobile_allowed': True, } template_values = qa.add_template_values(template_values, self.request) bingo([ 'struggling_videos_landing', 'suggested_activity_videos_landing', 'suggested_activity_videos_landing_binary', ]) self.render_jinja2_template('viewvideo.html', template_values)
def updateVideoAndPlaylistData(self): yt_service = gdata.youtube.service.YouTubeService() # Now that we run these queries from the App Engine servers, we need to # explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and # throttled by YouTube's "Too many request" quota yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ" yt_service.client_id = "n/a" video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id) video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all()) association_generation = int(Setting.last_youtube_sync_generation_start()) logging.info("Fetching playlists") playlist_start_index = 1 playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index) while len(playlist_feed.entry) > 0: for playlist in playlist_feed.entry: logging.info("Playlist: %s", playlist.id.text) playlist_id = playlist.id.text.replace('http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/', '') playlist_uri = playlist.id.text.replace('users/KhanAcademyHebrew/', '') query = Playlist.all() query.filter('youtube_id =', playlist_id) playlist_data = query.get() if not playlist_data: playlist_data = Playlist(youtube_id=playlist_id) logging.info('Creating Playlist: %s', playlist.title.text) playlist_data.url = playlist_uri playlist_data.title = playlist.title.text.decode("utf-8") playlist_data.description = playlist.description.text.decode("utf-8") playlist_data.tags = [] for category in playlist.category: if "tags.cat" in category.scheme: playlist_data.tags.append(category.term) playlist_data.put() for i in range(0, 10): start_index = i * 50 + 1 video_feed = yt_service.GetYouTubePlaylistVideoFeed(uri=playlist_uri + '?start-index=' + str(start_index) + '&max-results=50') video_data_list = [] if len(video_feed.entry) <= 0: # No more videos in playlist break for video in video_feed.entry: if not video.media.player: logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media) continue video_id = cgi.parse_qs(urlparse(video.media.player.url).query)['v'][0].decode('utf-8') video_data = None if video_youtube_id_dict.has_key(video_id): video_data = video_youtube_id_dict[video_id] logging.info('Found Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) if not video_data: video_data = Video(youtube_id=video_id) logging.info('Creating Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) video_data.title = video.media.title.text.decode('utf-8') video_data.url = video.media.player.url.decode('utf-8') video_data.duration = int(video.media.duration.seconds) if video.statistics: video_data.views = int(video.statistics.view_count) if video.media.description.text is not None: video_data.description = video.media.description.text.decode('utf-8') else: video_data.decription = ' ' if video.media.keywords.text: video_data.keywords = video.media.keywords.text.decode('utf-8') else: video_data.keywords = '' video_data.position = video.position video_data_list.append(video_data) db.put(video_data_list) playlist_videos = [] for video_data in video_data_list: playlist_video = None if video_playlist_key_dict.has_key(playlist_data.key()): if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()): playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()] if not playlist_video: playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key()) logging.info('Creating VideoPlaylist: %s, %s', playlist_data.title, video_data.title) else: logging.info('Updating VideoPlaylist: %s, %s', playlist_video.playlist.title, playlist_video.video.title) playlist_video.last_live_association_generation = association_generation playlist_video.video_position = int(video_data.position.text) playlist_videos.append(playlist_video) db.put(playlist_videos) # Check next set of playlists playlist_start_index += 50 playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
def _task_handler(uid, task_id=0, cursor=None, report=None): """Task chain for fetching subtitles from the Universal Subtitles API It processes Video models in batches of BATCH_SIZE by fetching the English subtitles via an HTTP API call. This job runs regularly so fetch failures are fixed from run-to-run. Fetch failures are logged and suppressed as the task marches on. Errors include URL fetch timeouts, subtitles put failures, and response decoding failures. HTTP redirects indicate that the code needs updating to a new API endpoint. They are detected and reported separately. """ query = Video.all() query.with_cursor(cursor) videos = query.fetch(BATCH_SIZE) if report is None: report = dict(REPORT_TEMPLATE) VideoSubtitlesFetchReport(key_name=uid, **report).put() # Asynchronously fetch. We'll rate-limit by fetching BATCH_SIZE subtitles # at each DEFER_SECONDS interval rpcs = [] for video in videos: url = UNISUBS_URL % urllib.quote(YOUTUBE_URL % video.youtube_id) rpc = urlfetch.create_rpc(deadline=TIMEOUT_SECONDS) urlfetch.make_fetch_call(rpc, url) rpcs.append((video.youtube_id, rpc)) report['fetches'] += 1 # Process asynchronous fetches for youtube_id, rpc in rpcs: lang = 'en' key_name = VideoSubtitles.get_key_name(lang, youtube_id) try: resp = rpc.get_result() if resp.status_code != 200: raise RuntimeError('status code: %s' % resp.status_code) if resp.final_url: logging.warn('%s redirect to %s' % (key_name, resp.final_url)) report['redirects'] += 1 json = resp.content.decode('utf-8') # Only update stale records current = VideoSubtitles.get_by_key_name(key_name) if not current or current.json != json: new = VideoSubtitles(key_name=key_name, youtube_id=youtube_id, language=lang, json=json) new.put() report['writes'] += 1 else: logging.info('%s content already up-to-date' % key_name) except Exception, e: logging.error('%s subtitles fetch failed: %s' % (key_name, e)) report['errors'] += 1
def library_content_html(): # No cache found -- regenerate HTML smart_history = getSmartHistoryContent() all_playlists = [] dict_videos = {} dict_videos_counted = {} dict_playlists = {} dict_playlists_by_title = {} dict_video_playlists = {} async_queries = [ Video.all(), Playlist.all(), VideoPlaylist.all().filter('live_association = ', True).order('video_position'), ] results = util.async_queries(async_queries) for video in results[0].get_result(): dict_videos[video.key()] = video for playlist in results[1].get_result(): dict_playlists[playlist.key()] = playlist if playlist.title in topics_list: dict_playlists_by_title[playlist.title] = playlist for video_playlist in results[2].get_result(): playlist_key = VideoPlaylist.playlist.get_value_for_datastore(video_playlist) video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist) if dict_videos.has_key(video_key) and dict_playlists.has_key(playlist_key): video = dict_videos[video_key] playlist = dict_playlists[playlist_key] fast_video_playlist_dict = {"video":video, "playlist":playlist} if dict_video_playlists.has_key(playlist_key): dict_video_playlists[playlist_key].append(fast_video_playlist_dict) else: dict_video_playlists[playlist_key] = [fast_video_playlist_dict] if dict_playlists_by_title.has_key(playlist.title): # Only count videos in topics_list dict_videos_counted[video.youtube_id] = True # Update count of all distinct videos associated w/ a live playlist Setting.count_videos(len(dict_videos_counted.keys())) for topic in topics_list: if topic in dict_playlists_by_title: playlist = dict_playlists_by_title[topic] playlist_key = playlist.key() playlist_videos = dict_video_playlists.get(playlist_key) or [] if not playlist_videos: logging.error('Playlist %s has no videos!', playlist.title) playlist_data = { 'title': topic, 'topic': topic, 'playlist': playlist, 'videos': playlist_videos, 'next': None } all_playlists.append(playlist_data) playlist_data_prev = None for playlist_data in all_playlists: if playlist_data_prev: playlist_data_prev['next'] = playlist_data playlist_data_prev = playlist_data # Separating out the columns because the formatting is a little different on each column template_values = { 'App' : App, 'all_playlists': all_playlists, 'smart_history': smart_history, } html = shared_jinja.get().render_template("library_content_template.html", **template_values) # Set shared date of last generated content Setting.cached_library_content_date(str(datetime.datetime.now())) return html
def get(self): from exercises import attempt_problem login_user = UserData.current() exercises_list = [exercise for exercise in Exercise.all()] videos_list = [video for video in Video.all()] user_count = self.request_int('users', 5) for user_id in xrange(0, user_count): # Create a new user first_name = random.choice(CreateRandomGoalData.first_names) last_name = random.choice(CreateRandomGoalData.last_names) nickname = "%s %s" % (first_name, last_name) email = 'test_%i@automatedrandomdata' % user_id user = users.User(email) logging.info("Creating user %s: (%i/%i)" % (nickname, user_id + 1, user_count)) user_data = UserData.get_or_insert( key_name="test_user_%i" % user_id, user=user, current_user=user, user_id=str(user_id), moderator=False, last_login=datetime.now(), proficient_exercises=[], suggested_exercises=[], need_to_reassess=True, points=0, coaches=[login_user.user_email], user_email=email, user_nickname=nickname, ) user_data.put() # Delete user exercise & video progress query = UserExercise.all() query.filter('user = '******'user = '******'type': 'GoalObjectiveExerciseProficiency', 'exercise': random.choice(exercises_list)}) for objective in xrange(1, random.randint(2, 4)): obj_descriptors.append({ 'type': 'GoalObjectiveWatchVideo', 'video': random.choice(videos_list)}) title = first_name + "'s Goal #" + str(goal_idx) logging.info("Creating goal " + title) objectives = GoalObjective.from_descriptors(obj_descriptors, user_data) goal = Goal(parent=user_data, title=title, objectives=objectives) user_data.save_goal(goal) for objective in obj_descriptors: if objective['type'] == 'GoalObjectiveExerciseProficiency': user_exercise = user_data.get_or_insert_exercise( objective['exercise']) chooser = random.randint(1, 120) if chooser >= 60: continue elif chooser > 15: count = 1 hints = 0 elif chooser < 7: count = 20 hints = 0 else: count = 25 hints = 1 logging.info( "Starting exercise: %s (%i problems, %i hints)" % (objective['exercise'].name, count, hints * count)) for i in xrange(1, count): attempt_problem(user_data, user_exercise, i, 1, 'TEST', 'TEST', 'TEST', True, hints, 0, False, "TEST", 'TEST', '0.0.0.0') elif objective['type'] == 'GoalObjectiveWatchVideo': seconds = random.randint(1, 1200) logging.info("Watching %i seconds of video %s" % (seconds, objective['video'].title)) VideoLog.add_entry(user_data, objective['video'], seconds, 0, detect_cheat=False) self.response.out.write('OK')
def updateVideoAndPlaylistData(self): yt_service = YouTubeService() video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id) video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all()) association_generation = int(Setting.last_youtube_sync_generation_start()) logging.info("Fetching playlists") playlist_start_index = 1 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50" % playlist_start_index ) while len(playlist_feed.entry) > 0: for playlist in playlist_feed.entry: logging.info("Playlist: %s", playlist.id.text) playlist_id = playlist.id.text.replace( "http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/", "" ) playlist_uri = playlist.id.text.replace("users/KhanAcademyHebrew/", "") query = Playlist.all() query.filter("youtube_id =", playlist_id) playlist_data = query.get() if not playlist_data: playlist_data = Playlist(youtube_id=playlist_id) logging.info("Creating Playlist: %s", playlist.title.text) playlist_data.url = playlist_uri playlist_data.title = playlist.title.text.decode("utf-8") playlist_data.description = playlist.description.text.decode("utf-8") playlist_data.tags = [] for category in playlist.category: if "tags.cat" in category.scheme: playlist_data.tags.append(category.term) playlist_data.put() for i in range(0, 10): start_index = i * 50 + 1 video_feed = yt_service.GetYouTubePlaylistVideoFeed( uri=playlist_uri + "?start-index=" + str(start_index) + "&max-results=50" ) video_data_list = [] if len(video_feed.entry) <= 0: # No more videos in playlist break for video in video_feed.entry: if not video.media.player: logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media) continue video_id = cgi.parse_qs(urlparse(video.media.player.url).query)["v"][0].decode("utf-8") video_data = None if video_youtube_id_dict.has_key(video_id): video_data = video_youtube_id_dict[video_id] logging.info("Found Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id) if not video_data: video_data = Video(youtube_id=video_id) logging.info("Creating Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id) video_data.title = video.media.title.text.decode("utf-8") video_data.url = video.media.player.url.decode("utf-8") video_data.duration = int(video.media.duration.seconds) if video.statistics: video_data.views = int(video.statistics.view_count) if video.media.description.text is not None: video_data.description = video.media.description.text.decode("utf-8") else: video_data.decription = " " if video.media.keywords.text: video_data.keywords = video.media.keywords.text.decode("utf-8") else: video_data.keywords = "" video_data.position = video.position video_data_list.append(video_data) db.put(video_data_list) playlist_videos = [] for video_data in video_data_list: playlist_video = None if video_playlist_key_dict.has_key(playlist_data.key()): if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()): playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()] if not playlist_video: playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key()) logging.info("Creating VideoPlaylist: %s, %s", playlist_data.title, video_data.title) else: logging.info( "Updating VideoPlaylist: %s, %s", playlist_video.playlist.title, playlist_video.video.title, ) playlist_video.last_live_association_generation = association_generation playlist_video.video_position = int(video_data.position.text) playlist_videos.append(playlist_video) db.put(playlist_videos) # Check next set of playlists playlist_start_index += 50 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50" % playlist_start_index )