def playlist_title_dicts():
return map(lambda playlist: {
"title": playlist.title,
"key": str(playlist.key()),
"ka_url": playlist.relative_url, # remove once js clients update
"url": playlist.relative_url
}, Playlist.all())
def get(self, request):
songs = Song.all_by_id()
playlists = Playlist.all()
context = {'songs': songs,
'playlists': playlists,
}
return render(request, 'music/index.html/', context)
def get_playlist_videos(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
query = VideoPlaylist.all()
query.filter('playlist =', playlist)
query.filter('live_association = ', True)
query.order('video_position')
videos = []
for pv in query.fetch(500):
v = pv.video
video_dict = {
'youtube_id':
v.youtube_id,
'youtube_url':
v.url,
'title':
v.title,
'description':
v.description,
'keywords':
v.keywords,
'readable_id':
v.readable_id,
'ka_url':
"http://www.khanacademy.org/video/%s?playlist=%s" %
(v.readable_id, urllib.quote_plus(playlist_title)),
'video_position':
pv.video_position,
'views':
v.views,
}
videos.append(video_dict)
return json.dumps(videos, indent=4)
def playlist_title_dicts():
return map(
lambda playlist: {
"title": playlist.title,
"key": str(playlist.key()),
"ka_url": "/#%s" % slugify(playlist.title.lower())
}, Playlist.all())
def playlist_title_dicts():
return map(
lambda playlist: {
"title": playlist.title,
"key": str(playlist.key()),
"ka_url": playlist.relative_url, # remove once js clients update
"url": playlist.relative_url
},
Playlist.all())
def playlist_content_html():
"""" Returns the HTML for the structure of the playlists as they will be
populated ont he homepage. Does not actually contain the list of video
names as those are filled in later asynchronously via the cache.
"""
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
dict_playlists_by_title = {}
all_playlists = []
for playlist in Playlist.all():
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
video_count = playlist.get_video_count()
# 3 columns, 18px per row. This must be updated in conjunction
# with code in homepage.js
height = math.ceil(video_count / 3) * 18
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'list_height': height,
'next': None,
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
timestamp = time.time()
template_values = {
'App': App,
'all_playlists': all_playlists,
'smart_history': smart_history,
# convert timestamp to a nice integer for the JS
'timestamp': int(round(timestamp * 1000)),
}
html = shared_jinja.get().render_template("library_playlist_template.html",
**template_values)
Setting.cached_playlist_content_date(
str(datetime.datetime.fromtimestamp(timestamp)))
return html
def playlist_content_html():
"""" Returns the HTML for the structure of the playlists as they will be
populated ont he homepage. Does not actually contain the list of video
names as those are filled in later asynchronously via the cache.
"""
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
dict_playlists_by_title = {}
all_playlists = []
for playlist in Playlist.all():
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
video_count = playlist.get_video_count()
# 3 columns, 18px per row. This must be updated in conjunction
# with code in homepage.js
height = math.ceil(video_count / 3) * 18
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'list_height': height,
'next': None,
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
timestamp = time.time()
template_values = {
'App' : App,
'all_playlists': all_playlists,
'smart_history': smart_history,
# convert timestamp to a nice integer for the JS
'timestamp': int(round(timestamp * 1000)),
}
html = shared_jinja.get().render_template("library_playlist_template.html",
**template_values)
Setting.cached_playlist_content_date(
str(datetime.datetime.fromtimestamp(timestamp)))
return html
def copyTopicsToPlaylist(self):
topic_list = Topic.get_content_topics()
for topic in topic_list:
playlist = Playlist.all().filter("title =",
topic.standalone_title).get()
if playlist:
logging.info("Copying topic " + topic.standalone_title +
" to playlist.")
child_keys = topic.child_keys
vps = VideoPlaylist.all().filter(
"playlist =",
playlist).order("video_position").fetch(10000)
playlist_keys = []
for vp in vps:
try:
playlist_keys.append(vp.video.key())
except db.ReferencePropertyResolveError:
logging.info(
"Found reference to missing video in VideoPlaylist!"
)
topic_keys = [
key for key in topic.child_keys if key.kind() == "Video"
]
if playlist_keys == topic_keys:
logging.info(
"Child keys identical. No changes will be made.")
else:
# logging.info("PLAYLIST: " + repr([str(key) for key in playlist_keys]))
# logging.info("TOPIC: " + repr([str(key) for key in topic_keys]))
logging.info("Deleting old VideoPlaylists...")
db.delete(vps)
vps = []
for i, child_key in enumerate(topic.child_keys):
if child_key.kind() == "Video":
vps.append(
VideoPlaylist(video=child_key,
playlist=playlist,
video_position=i,
live_association=True))
logging.info("Creating new VideoPlaylists...")
db.put(vps)
else:
logging.info("Playlist matching topic " +
topic.standalone_title + " not found.")
def load_videos(version, title=None):
root = Topic.get_by_id("root", version)
if title is None:
playlist = Playlist.all().order('title').get()
else:
playlist = Playlist.all().filter('title = ', title).get()
title = playlist.title
nextplaylist = Playlist.all().filter('title >', title).order('title').get()
if nextplaylist:
next_title = nextplaylist.title
playlists = [playlist]
# playlists = Playlist.all().order('title').fetch(100000)
for i, p in enumerate(playlists):
videos = p.get_videos()
content_keys = [v.key() for v in videos]
added = 0
for i, v in enumerate(videos):
for e in v.related_exercises():
if e.key() not in content_keys:
content_keys.insert(i + added, e.key())
added += 1
topic = Topic.insert(title=p.title,
parent=root,
description=p.description,
tags=p.tags,
child_keys=content_keys)
logging.info("loading " + title)
if nextplaylist:
deferred.defer(load_videos, version, next_title)
else:
deferred.defer(hide_topics, version)
def load_videos(version, title=None):
root = Topic.get_by_id("root", version)
if title is None:
playlist = Playlist.all().order('title').get()
else:
playlist = Playlist.all().filter('title = ', title).get()
title = playlist.title
nextplaylist = Playlist.all().filter('title >', title).order('title').get()
if nextplaylist:
next_title = nextplaylist.title
playlists = [playlist]
# playlists = Playlist.all().order('title').fetch(100000)
for i, p in enumerate(playlists):
videos = p.get_videos()
content_keys = [v.key() for v in videos]
added = 0
for i, v in enumerate(videos):
for e in v.related_exercises():
if e.key() not in content_keys:
content_keys.insert(i + added, e.key())
added += 1
topic = Topic.insert(title=p.title,
parent=root,
description=p.description,
tags=p.tags,
child_keys=content_keys)
logging.info("loading " + title)
if nextplaylist:
deferred.defer(load_videos, version, next_title)
else:
deferred.defer(hide_topics, version)
def get_playlists():
playlists = []
for playlist_title in all_topics_list:
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
playlist_dict = {'youtube_id': playlist.youtube_id,
'youtube_url': playlist.url,
'title': playlist.title,
'description': playlist.description,
'api_url': "http://www.khanacademy.org/api/playlistvideos?playlist=%s" % (urllib.quote_plus(playlist_title)),
}
playlists.append(playlist_dict)
return json.dumps(playlists, indent=4)
def get_playlist_videos_json(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
video_query = Video.all()
video_query.filter('playlists = ', playlist_title)
video_key_dict = Video.get_dict(video_query, lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('playlist =', playlist)
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
return json.dumps(get_playlist_video_api_dicts(playlist, video_key_dict, video_playlist_key_dict), indent=4)
def get(self):
playlist_title = "SAT Preparation"
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
query = VideoPlaylist.all()
query.filter('playlist =', playlist)
query.filter('live_association = ', True) #need to change this to true once I'm done with all of my hacks
query.order('video_position')
playlist_videos = query.fetch(500)
template_values = {
'videos': playlist_videos,
}
self.render_jinja2_template('sat.html', template_values)
def get_playlist_videos_json(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
video_query = Video.all()
video_query.filter('playlists = ', playlist_title)
video_key_dict = Video.get_dict(video_query, lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('playlist =', playlist)
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
return json.dumps(get_playlist_video_api_dicts(playlist, video_key_dict,
video_playlist_key_dict),
indent=4)
def getalltracks():
playlists = Playlist.all()
response = {}
for playlist in playlists:
pl = {}
tracks = {}
i = 1
for track in playlist.tracks:
trackobj = Track.get(track)
tracks[i] = {'url':trackobj.url, 'type':'sc'} #FIXME 'sc' for now, change to track.type
i += 1
pl['tracks'] = tracks
response[playlist.name] = pl
return jsonify(response)
def copyTopicsToPlaylist(self):
topic_list = Topic.get_content_topics()
for topic in topic_list:
playlist = Playlist.all().filter("title =", topic.standalone_title).get()
if playlist:
logging.info("Copying topic " + topic.standalone_title + " to playlist.")
child_keys = topic.child_keys
vps = VideoPlaylist.all().filter("playlist =", playlist).order("video_position").fetch(10000)
playlist_keys = []
for vp in vps:
try:
playlist_keys.append(vp.video.key())
except db.ReferencePropertyResolveError:
logging.info("Found reference to missing video in VideoPlaylist!")
topic_keys = [key for key in topic.child_keys if key.kind() == "Video"]
if playlist_keys == topic_keys:
logging.info("Child keys identical. No changes will be made.")
else:
# logging.info("PLAYLIST: " + repr([str(key) for key in playlist_keys]))
# logging.info("TOPIC: " + repr([str(key) for key in topic_keys]))
logging.info("Deleting old VideoPlaylists...")
db.delete(vps)
vps = []
for i, child_key in enumerate(topic.child_keys):
if child_key.kind() == "Video":
vps.append(VideoPlaylist(
video=child_key,
playlist=playlist,
video_position=i,
live_association = True
))
logging.info("Creating new VideoPlaylists...")
db.put(vps)
else:
logging.info("Playlist matching topic " + topic.standalone_title + " not found.")
def get_playlists():
playlists = []
for playlist_title in all_topics_list:
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
playlist_dict = {
'youtube_id':
playlist.youtube_id,
'youtube_url':
playlist.url,
'title':
playlist.title,
'description':
playlist.description,
'api_url':
"http://www.khanacademy.org/api/playlistvideos?playlist=%s" %
(urllib.quote_plus(playlist_title)),
}
playlists.append(playlist_dict)
return json.dumps(playlists, indent=4)
def get_playlist_videos(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
query = VideoPlaylist.all()
query.filter('playlist =', playlist)
query.filter('live_association = ', True)
query.order('video_position')
videos = []
for pv in query.fetch(500):
v = pv.video
video_dict = {'youtube_id': v.youtube_id,
'youtube_url': v.url,
'title': v.title,
'description': v.description,
'keywords': v.keywords,
'readable_id': v.readable_id,
'ka_url': "http://www.khanacademy.org/video/%s?playlist=%s" % (v.readable_id, urllib.quote_plus(playlist_title)),
'video_position': pv.video_position,
'views': v.views,
}
videos.append(video_dict)
return json.dumps(videos, indent=4)
def playlist_title_dicts():
return map(lambda playlist: {"title": playlist.title, "url": "/#%s" % playlist.title}, Playlist.all())
def playlist_title_dicts():
return map(
lambda playlist: {
"title": playlist.title,
"url": "/#%s" % playlist.title
}, Playlist.all())
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ',
True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(
video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(
playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video": video, "playlist": playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(
fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App': App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html",
**template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def get(self, readable_id=""):
# This method displays a video in the context of a particular playlist.
# To do that we first need to find the appropriate playlist. If we aren't
# given the playlist title in a query param, we need to find a playlist that
# the video is a part of. That requires finding the video, given it readable_id
# or, to support old URLs, it's youtube_id.
video = None
playlist = None
video_id = self.request.get('v')
playlist_title = self.request_string('playlist', default="") or self.request_string('p', default="")
readable_id = urllib.unquote(readable_id)
readable_id = re.sub('-+$', '', readable_id) # remove any trailing dashes (see issue 1140)
# If either the readable_id or playlist title is missing,
# redirect to the canonical URL that contains them
redirect_to_canonical_url = False
if video_id: # Support for old links
query = Video.all()
query.filter('youtube_id =', video_id)
video = query.get()
if not video:
raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id)
readable_id = video.readable_id
playlist = video.first_playlist()
if not playlist:
raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id)
redirect_to_canonical_url = True
if playlist_title is not None and len(playlist_title) > 0:
query = Playlist.all().filter('title =', playlist_title)
key_id = 0
for p in query:
if p.key().id() > key_id and not p.youtube_id.endswith('_player'):
playlist = p
key_id = p.key().id()
# If a playlist_title wasn't specified or the specified playlist wasn't found
# use the first playlist for the requested video.
if playlist is None:
# Get video by readable_id just to get the first playlist for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
playlist = video.first_playlist()
if not playlist:
raise MissingVideoException("Missing video '%s'" % readable_id)
redirect_to_canonical_url = True
exid = self.request_string('exid', default=None)
if redirect_to_canonical_url:
qs = {'playlist': playlist.title}
if exid:
qs['exid'] = exid
urlpath = "/video/%s" % urllib.quote(readable_id)
url = urlparse.urlunparse(('', '', urlpath, '', urllib.urlencode(qs), ''))
self.redirect(url, True)
return
# If we got here, we have a readable_id and a playlist_title, so we can display
# the playlist and the video in it that has the readable_id. Note that we don't
# query the Video entities for one with the requested readable_id because in some
# cases there are multiple Video objects in the datastore with the same readable_id
# (e.g. there are 2 "Order of Operations" videos).
videos = VideoPlaylist.get_cached_videos_for_playlist(playlist)
previous_video = None
next_video = None
for v in videos:
if v.readable_id == readable_id:
v.selected = 'selected'
video = v
elif video is None:
previous_video = v
elif next_video is None:
next_video = v
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
if App.offline_mode:
video_path = "/videos/" + get_mangled_playlist_name(playlist_title) + "/" + video.readable_id + ".flv"
else:
video_path = video.download_video_url()
if video.description == video.title:
video.description = None
related_exercises = video.related_exercises()
button_top_exercise = None
if related_exercises:
def ex_to_dict(exercise):
return {
'name': exercise.display_name,
'url': exercise.relative_url,
}
button_top_exercise = ex_to_dict(related_exercises[0])
user_video = UserVideo.get_for_video_and_user_data(video, UserData.current(), insert_if_missing=True)
awarded_points = 0
if user_video:
awarded_points = user_video.points
template_values = {
'playlist': playlist,
'video': video,
'videos': videos,
'video_path': video_path,
'video_points_base': consts.VIDEO_POINTS_BASE,
'button_top_exercise': button_top_exercise,
'related_exercises': [], # disabled for now
'previous_video': previous_video,
'next_video': next_video,
'selected_nav_link': 'watch',
'awarded_points': awarded_points,
'issue_labels': ('Component-Videos,Video-%s' % readable_id),
'author_profile': 'https://plus.google.com/103970106103092409324'
}
template_values = qa.add_template_values(template_values, self.request)
bingo(['struggling_videos_landing',
'homepage_restructure_videos_landing'])
self.render_jinja2_template('viewvideo.html', template_values)
def x():
playlist = Playlist.all().filter('name =', 'Godlista').get()
trackkeys = [i.key() for i in Track.all()]
playlist.tracks = trackkeys
playlist.put()
return str(playlist.tracks)
def get_all_topic_playlists():
playlists = []
for playlist in Playlist.all().fetch(1000):
if playlist.title in all_topics_list:
playlists.append(playlist)
return playlists
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ', True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video":video, "playlist":playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App' : App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html", **template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def indexPlaylistData(self):
playlists = Playlist.all().fetch(10000)
for playlist in playlists:
playlist.index()
playlist.indexed_title_changed()
def updateVideoAndPlaylistData(self):
yt_service = gdata.youtube.service.YouTubeService()
# Now that we run these queries from the App Engine servers, we need to
# explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and
# throttled by YouTube's "Too many request" quota
yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ"
yt_service.client_id = "n/a"
video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all())
association_generation = int(Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace('http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/', '')
playlist_uri = playlist.id.text.replace('users/KhanAcademyHebrew/', '')
query = Playlist.all()
query.filter('youtube_id =', playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info('Creating Playlist: %s', playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode("utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(uri=playlist_uri + '?start-index=' + str(start_index) + '&max-results=50')
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media)
continue
video_id = cgi.parse_qs(urlparse(video.media.player.url).query)['v'][0].decode('utf-8')
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info('Found Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info('Creating Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id)
video_data.title = video.media.title.text.decode('utf-8')
video_data.url = video.media.player.url.decode('utf-8')
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode('utf-8')
else:
video_data.decription = ' '
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode('utf-8')
else:
video_data.keywords = ''
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(playlist_data.key()):
if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()):
playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key())
logging.info('Creating VideoPlaylist: %s, %s', playlist_data.title, video_data.title)
else:
logging.info('Updating VideoPlaylist: %s, %s', playlist_video.playlist.title, playlist_video.video.title)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
def updateVideoAndPlaylistData(self):
yt_service = YouTubeService()
video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all())
association_generation = int(Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50"
% playlist_start_index
)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace(
"http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/", ""
)
playlist_uri = playlist.id.text.replace("users/KhanAcademyHebrew/", "")
query = Playlist.all()
query.filter("youtube_id =", playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info("Creating Playlist: %s", playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode("utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(
uri=playlist_uri + "?start-index=" + str(start_index) + "&max-results=50"
)
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media)
continue
video_id = cgi.parse_qs(urlparse(video.media.player.url).query)["v"][0].decode("utf-8")
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info("Found Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info("Creating Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id)
video_data.title = video.media.title.text.decode("utf-8")
video_data.url = video.media.player.url.decode("utf-8")
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode("utf-8")
else:
video_data.decription = " "
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode("utf-8")
else:
video_data.keywords = ""
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(playlist_data.key()):
if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()):
playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key())
logging.info("Creating VideoPlaylist: %s, %s", playlist_data.title, video_data.title)
else:
logging.info(
"Updating VideoPlaylist: %s, %s",
playlist_video.playlist.title,
playlist_video.video.title,
)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50"
% playlist_start_index
)
def updateVideoAndPlaylistData(self):
yt_service = gdata.youtube.service.YouTubeService()
# Now that we run these queries from the App Engine servers, we need to
# explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and
# throttled by YouTube's "Too many request" quota
yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ"
yt_service.client_id = "n/a"
video_youtube_id_dict = Video.get_dict(Video.all(),
lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(
VideoPlaylist.all())
association_generation = int(
Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri=
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50'
% playlist_start_index)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace(
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/',
'')
playlist_uri = playlist.id.text.replace(
'users/KhanAcademyHebrew/', '')
query = Playlist.all()
query.filter('youtube_id =', playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info('Creating Playlist: %s', playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode(
"utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(
uri=playlist_uri + '?start-index=' + str(start_index) +
'&max-results=50')
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning(
"Could not parse video - skipping... (%s, %s)",
video, video.media)
continue
video_id = cgi.parse_qs(
urlparse(
video.media.player.url).query)['v'][0].decode(
'utf-8')
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info(
'Found Video: %s (%s)',
video.media.title.text.decode('utf-8'),
video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info(
'Creating Video: %s (%s)',
video.media.title.text.decode('utf-8'),
video_id)
video_data.title = video.media.title.text.decode(
'utf-8')
video_data.url = video.media.player.url.decode('utf-8')
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode(
'utf-8')
else:
video_data.decription = ' '
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode(
'utf-8')
else:
video_data.keywords = ''
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(
playlist_data.key()):
if video_playlist_key_dict[
playlist_data.key()].has_key(
video_data.key()):
playlist_video = video_playlist_key_dict[
playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(
playlist=playlist_data.key(),
video=video_data.key())
logging.info('Creating VideoPlaylist: %s, %s',
playlist_data.title, video_data.title)
else:
logging.info('Updating VideoPlaylist: %s, %s',
playlist_video.playlist.title,
playlist_video.video.title)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(
video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri=
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50'
% playlist_start_index)
def get_all_topic_playlists():
playlists = []
for playlist in Playlist.all().fetch(1000):
if playlist.title in all_topics_list:
playlists.append(playlist)
return playlists
def indexPlaylistData(self):
playlists = Playlist.all().fetch(10000)
for playlist in playlists:
playlist.index()
playlist.indexed_title_changed()
