def copyTopicsToPlaylist(self): topic_list = Topic.get_content_topics() for topic in topic_list: playlist = Playlist.all().filter("title =", topic.standalone_title).get() if playlist: logging.info("Copying topic " + topic.standalone_title + " to playlist.") child_keys = topic.child_keys vps = VideoPlaylist.all().filter( "playlist =", playlist).order("video_position").fetch(10000) playlist_keys = [] for vp in vps: try: playlist_keys.append(vp.video.key()) except db.ReferencePropertyResolveError: logging.info( "Found reference to missing video in VideoPlaylist!" ) topic_keys = [ key for key in topic.child_keys if key.kind() == "Video" ] if playlist_keys == topic_keys: logging.info( "Child keys identical. No changes will be made.") else: # logging.info("PLAYLIST: " + repr([str(key) for key in playlist_keys])) # logging.info("TOPIC: " + repr([str(key) for key in topic_keys])) logging.info("Deleting old VideoPlaylists...") db.delete(vps) vps = [] for i, child_key in enumerate(topic.child_keys): if child_key.kind() == "Video": vps.append( VideoPlaylist(video=child_key, playlist=playlist, video_position=i, live_association=True)) logging.info("Creating new VideoPlaylists...") db.put(vps) else: logging.info("Playlist matching topic " + topic.standalone_title + " not found.")
def updateVideoAndPlaylistData(self): yt_service = gdata.youtube.service.YouTubeService() # Now that we run these queries from the App Engine servers, we need to # explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and # throttled by YouTube's "Too many request" quota yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ" yt_service.client_id = "n/a" video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id) video_playlist_key_dict = VideoPlaylist.get_key_dict( VideoPlaylist.all()) association_generation = int( Setting.last_youtube_sync_generation_start()) logging.info("Fetching playlists") playlist_start_index = 1 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri= 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index) while len(playlist_feed.entry) > 0: for playlist in playlist_feed.entry: logging.info("Playlist: %s", playlist.id.text) playlist_id = playlist.id.text.replace( 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/', '') playlist_uri = playlist.id.text.replace( 'users/KhanAcademyHebrew/', '') query = Playlist.all() query.filter('youtube_id =', playlist_id) playlist_data = query.get() if not playlist_data: playlist_data = Playlist(youtube_id=playlist_id) logging.info('Creating Playlist: %s', playlist.title.text) playlist_data.url = playlist_uri playlist_data.title = playlist.title.text.decode("utf-8") playlist_data.description = playlist.description.text.decode( "utf-8") playlist_data.tags = [] for category in playlist.category: if "tags.cat" in category.scheme: playlist_data.tags.append(category.term) playlist_data.put() for i in range(0, 10): start_index = i * 50 + 1 video_feed = yt_service.GetYouTubePlaylistVideoFeed( uri=playlist_uri + '?start-index=' + str(start_index) + '&max-results=50') video_data_list = [] if len(video_feed.entry) <= 0: # No more videos in playlist break for video in video_feed.entry: if not video.media.player: logging.warning( "Could not parse video - skipping... (%s, %s)", video, video.media) continue video_id = cgi.parse_qs( urlparse( video.media.player.url).query)['v'][0].decode( 'utf-8') video_data = None if video_youtube_id_dict.has_key(video_id): video_data = video_youtube_id_dict[video_id] logging.info( 'Found Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) if not video_data: video_data = Video(youtube_id=video_id) logging.info( 'Creating Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id) video_data.title = video.media.title.text.decode( 'utf-8') video_data.url = video.media.player.url.decode('utf-8') video_data.duration = int(video.media.duration.seconds) if video.statistics: video_data.views = int(video.statistics.view_count) if video.media.description.text is not None: video_data.description = video.media.description.text.decode( 'utf-8') else: video_data.decription = ' ' if video.media.keywords.text: video_data.keywords = video.media.keywords.text.decode( 'utf-8') else: video_data.keywords = '' video_data.position = video.position video_data_list.append(video_data) db.put(video_data_list) playlist_videos = [] for video_data in video_data_list: playlist_video = None if video_playlist_key_dict.has_key( playlist_data.key()): if video_playlist_key_dict[ playlist_data.key()].has_key( video_data.key()): playlist_video = video_playlist_key_dict[ playlist_data.key()][video_data.key()] if not playlist_video: playlist_video = VideoPlaylist( playlist=playlist_data.key(), video=video_data.key()) logging.info('Creating VideoPlaylist: %s, %s', playlist_data.title, video_data.title) else: logging.info('Updating VideoPlaylist: %s, %s', playlist_video.playlist.title, playlist_video.video.title) playlist_video.last_live_association_generation = association_generation playlist_video.video_position = int( video_data.position.text) playlist_videos.append(playlist_video) db.put(playlist_videos) # Check next set of playlists playlist_start_index += 50 playlist_feed = yt_service.GetYouTubePlaylistFeed( uri= 'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)