def updateVideoAndPlaylistReadableNames(self):
# Makes sure every video and playlist has a unique "name" that can be used in URLs
query = Video.all()
all_videos = query.fetch(100000)
for video in all_videos:
potential_id = re.sub('[^a-z0-9]', '-', video.title.lower())
potential_id = re.sub(
'-+$', '',
potential_id) # remove any trailing dashes (see issue 1140)
potential_id = re.sub(
'^-+', '',
potential_id) # remove any leading dashes (see issue 1526)
if video.readable_id == potential_id: # id is unchanged
continue
number_to_add = 0
current_id = potential_id
while True:
query = Video.all()
query.filter('readable_id=', current_id)
if (query.get() is
None): #id is unique so use it and break out
video.readable_id = current_id
video.put()
break
else: # id is not unique so will have to go through loop again
number_to_add += 1
current_id = potential_id + '-' + number_to_add
def update(self, feedback):
orig_video = feedback.video()
if orig_video == None or type(orig_video).__name__ != "Video":
return False
readable_id = orig_video.readable_id
query = Video.all()
query.filter('readable_id =', readable_id)
# The database currently contains multiple Video objects for a particular
# video. Some are old. Some are due to a YouTube sync where the youtube urls
# changed and our code was producing youtube_ids that ended with '_player'.
# This hack gets the most recent valid Video object.
key_id = 0
for v in query:
if v.key().id() > key_id and not v.youtube_id.endswith('_player'):
video = v
key_id = v.key().id()
# End of hack
if video is not None and video.key() != orig_video.key():
logging.info("Retargeting Feedback %s from Video %s to Video %s",
feedback.key().id(),
orig_video.key().id(),
video.key().id())
feedback.targets[0] = video.key()
return True
else:
return False
def video_title_dicts():
live_video_dict = {}
for video_playlist in VideoPlaylist.all().filter('live_association = ', True):
live_video_dict[VideoPlaylist.video.get_value_for_datastore(video_playlist)] = True
live_videos = filter(lambda video: video.key() in live_video_dict, Video.all())
return map(lambda video: {"title": video.title, "url": "/video/%s" % video.readable_id}, live_videos)
def update_video(self, video_youtube_id):
v = Video.all().filter('youtube_id =', video_youtube_id).get()
if v is not None:
if v.key() not in self.videos:
self.videos.append(v.key())
return True
else:
logging.debug("Youtube ID %s not in datastore" % video_youtube_id)
def test_derive_key_name_from_video(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
videos = Video.all().fetch(BATCH_SIZE)
for v in videos:
key = VideoSubtitles.get_key_name('en', v.youtube_id)
subs = VideoSubtitles.get_by_key_name(key)
self.assertIsNotNone(subs)
def updateVideoAndPlaylistReadableNames(self):
# Makes sure every video and playlist has a unique "name" that can be used in URLs
query = Video.all()
all_videos = query.fetch(100000)
for video in all_videos:
potential_id = re.sub('[^a-z0-9]', '-', video.title.lower());
potential_id = re.sub('-+$', '', potential_id) # remove any trailing dashes (see issue 1140)
potential_id = re.sub('^-+', '', potential_id) # remove any leading dashes (see issue 1526)
if video.readable_id == potential_id: # id is unchanged
continue
number_to_add = 0
current_id = potential_id
while True:
query = Video.all()
query.filter('readable_id=', current_id)
if (query.get() is None): #id is unique so use it and break out
video.readable_id = current_id
video.put()
break
else: # id is not unique so will have to go through loop again
number_to_add+=1
current_id = potential_id+'-'+number_to_add
def get(self):
user_data = UserData.current()
video_points_total = 0
if user_data:
video = None
key_str = self.request_string("video_key")
if key_str:
key = db.Key(key_str)
app_id = os.environ['APPLICATION_ID']
if key.app() != app_id:
new_key = db.Key.from_path(key.kind(),
key.id() or key.name(),
_app=app_id)
logging.warning(
"Key '%s' had invalid app_id '%s'. Changed to new key '%s'",
str(key), key.app(), str(new_key))
key = new_key
video = db.get(key)
else:
youtube_id = self.request_string("youtube_id")
if youtube_id:
video = Video.all().filter('youtube_id =',
youtube_id).get()
if video:
# Seconds watched is restricted by both the scrubber's position
# and the amount of time spent on the video page
# so we know how *much* of each video each student has watched
seconds_watched = int(
self.request_float("seconds_watched", default=0))
last_second_watched = int(
self.request_float("last_second_watched", default=0))
user_video, video_log, video_points_total = VideoLog.add_entry(
user_data, video, seconds_watched, last_second_watched)
user_points_html = self.render_jinja2_template_to_string(
"user_points_only.html", user_points(user_data))
json = simplejson.dumps(
{
"user_points_html": user_points_html,
"video_points": video_points_total
},
ensure_ascii=False)
self.response.out.write(json)
def video_title_dicts():
live_video_dict = {}
for video_playlist in VideoPlaylist.all().filter('live_association = ',
True):
live_video_dict[VideoPlaylist.video.get_value_for_datastore(
video_playlist)] = True
live_videos = filter(lambda video: video.key() in live_video_dict,
Video.all())
return map(
lambda video: {
"title": video.title,
"url": "/video/%s" % video.readable_id
}, live_videos)
def get_playlist_videos_json(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
video_query = Video.all()
video_query.filter('playlists = ', playlist_title)
video_key_dict = Video.get_dict(video_query, lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('playlist =', playlist)
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
return json.dumps(get_playlist_video_api_dicts(playlist, video_key_dict, video_playlist_key_dict), indent=4)
def get_video_library_json_compressed():
playlist_api_dicts = []
playlists = get_all_topic_playlists()
video_key_dict = Video.get_dict(Video.all(), lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
for playlist in playlists:
playlist_api_dict = JsonApiDict.playlist(playlist)
playlist_api_dict["videos"] = get_playlist_video_api_dicts(playlist, video_key_dict, video_playlist_key_dict)
playlist_api_dicts.append(playlist_api_dict)
# We compress this huge json payload so it'll fit in memcache
return zlib.compress(json.dumps(playlist_api_dicts, indent=4))
def test_process_next_batch_on_nonempty_cursor(self):
offset = 3
# these should be skipped, they'll DownloadError
for i in xrange(0, offset):
Video(youtube_id=str(i)).put()
# these should be downloaded
self._set_responses_xrange(offset, BATCH_SIZE + offset)
query = Video.all()
query.fetch(offset)
cursor = query.cursor()
_task_handler('UUID', cursor=cursor)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def get_playlist_videos_json(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
video_query = Video.all()
video_query.filter('playlists = ', playlist_title)
video_key_dict = Video.get_dict(video_query, lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('playlist =', playlist)
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
return json.dumps(get_playlist_video_api_dicts(playlist, video_key_dict,
video_playlist_key_dict),
indent=4)
def get_video_library_json_compressed():
playlist_api_dicts = []
playlists = get_all_topic_playlists()
video_key_dict = Video.get_dict(Video.all(), lambda video: video.key())
video_playlist_query = VideoPlaylist.all()
video_playlist_query.filter('live_association =', True)
video_playlist_key_dict = VideoPlaylist.get_key_dict(video_playlist_query)
for playlist in playlists:
playlist_api_dict = JsonApiDict.playlist(playlist)
playlist_api_dict["videos"] = get_playlist_video_api_dicts(
playlist, video_key_dict, video_playlist_key_dict)
playlist_api_dicts.append(playlist_api_dict)
# We compress this huge json payload so it'll fit in memcache
return zlib.compress(json.dumps(playlist_api_dicts, indent=4))
def youtube_get_video_data_dict(youtube_id):
yt_service = third_party.gdata.youtube.service.YouTubeService()
# Now that we run these queries from the App Engine servers, we need to
# explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and
# throttled by YouTube's "Too many request" quota
yt_service.developer_key = "AI39si5NKByjOThtM6t1gnmg4N9HkzGPHHkVvRUybKuPG53277wY0Bs4zcvi-G4JiFioRs0738gaE01k1rX-_6-mIHp9jg1twQ"
yt_service.client_id = "n/a"
logging.info("trying to get info for youtube_id: %s" % youtube_id)
try:
video = yt_service.GetYouTubeVideoEntry(video_id=youtube_id)
except:
video = None
if video:
video_data = {"youtube_id" : youtube_id,
"title" : video.media.title.text.decode('utf-8'),
"url" : video.media.player.url.decode('utf-8'),
"duration" : int(video.media.duration.seconds)}
if video.statistics:
video_data["views"] = int(video.statistics.view_count)
video_data["description"] = (video.media.description.text or '').decode('utf-8')
video_data["keywords"] = (video.media.keywords.text or '').decode('utf-8')
potential_id = re.sub('[^a-z0-9]', '-', video_data["title"].lower());
potential_id = re.sub('-+$', '', potential_id) # remove any trailing dashes (see issue 1140)
potential_id = re.sub('^-+', '', potential_id) # remove any leading dashes (see issue 1526)
number_to_add = 0
current_id = potential_id
while True:
query = Video.all()
query.filter('readable_id=', current_id)
if (query.get() is None): #id is unique so use it and break out
video_data["readable_id"] = current_id
break
else: # id is not unique so will have to go through loop again
number_to_add+=1
current_id = potential_id+'-'+number_to_add
return video_data
return None
def get(self):
user_data = UserData.current()
video_points_total = 0
if user_data:
video = None
key_str = self.request_string("video_key")
if key_str:
key = db.Key(key_str)
app_id = os.environ['APPLICATION_ID']
if key.app() != app_id:
new_key = db.Key.from_path(
key.kind(),
key.id() or key.name(),
_app=app_id)
logging.warning("Key '%s' had invalid app_id '%s'. Changed to new key '%s'", str(key), key.app(), str(new_key))
key = new_key
video = db.get(key)
else:
youtube_id = self.request_string("youtube_id")
if youtube_id:
video = Video.all().filter('youtube_id =', youtube_id).get()
if video:
# Seconds watched is restricted by both the scrubber's position
# and the amount of time spent on the video page
# so we know how *much* of each video each student has watched
seconds_watched = int(self.request_float("seconds_watched", default=0))
last_second_watched = int(self.request_float("last_second_watched", default=0))
user_video, video_log, video_points_total = VideoLog.add_entry(user_data, video, seconds_watched, last_second_watched)
user_points_html = self.render_jinja2_template_to_string("user_points_only.html", user_points(user_data))
json = simplejson.dumps({"user_points_html": user_points_html, "video_points": video_points_total}, ensure_ascii=False)
self.response.out.write(json)
def update(self, feedback):
orig_video = feedback.video()
if orig_video == None or type(orig_video).__name__ != "Video":
return False
readable_id = orig_video.readable_id
query = Video.all()
query.filter('readable_id =', readable_id)
# The database currently contains multiple Video objects for a particular
# video. Some are old. Some are due to a YouTube sync where the youtube urls
# changed and our code was producing youtube_ids that ended with '_player'.
# This hack gets the most recent valid Video object.
key_id = 0
for v in query:
if v.key().id() > key_id and not v.youtube_id.endswith('_player'):
video = v
key_id = v.key().id()
# End of hack
if video is not None and video.key() != orig_video.key():
logging.info("Retargeting Feedback %s from Video %s to Video %s", feedback.key().id(), orig_video.key().id(), video.key().id())
feedback.targets[0] = video.key()
return True
else:
return False
def indexVideoData(self):
videos = Video.all().fetch(10000)
for video in videos:
video.index()
video.indexed_title_changed()
def api_get_videos():
videos = Video.all()
api_response = jsonify(ok=True, result=videos)
return api_response
def updateVideoAndPlaylistData(self):
yt_service = gdata.youtube.service.YouTubeService()
# Now that we run these queries from the App Engine servers, we need to
# explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and
# throttled by YouTube's "Too many request" quota
yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ"
yt_service.client_id = "n/a"
video_youtube_id_dict = Video.get_dict(Video.all(),
lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(
VideoPlaylist.all())
association_generation = int(
Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri=
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50'
% playlist_start_index)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace(
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/',
'')
playlist_uri = playlist.id.text.replace(
'users/KhanAcademyHebrew/', '')
query = Playlist.all()
query.filter('youtube_id =', playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info('Creating Playlist: %s', playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode(
"utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(
uri=playlist_uri + '?start-index=' + str(start_index) +
'&max-results=50')
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning(
"Could not parse video - skipping... (%s, %s)",
video, video.media)
continue
video_id = cgi.parse_qs(
urlparse(
video.media.player.url).query)['v'][0].decode(
'utf-8')
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info(
'Found Video: %s (%s)',
video.media.title.text.decode('utf-8'),
video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info(
'Creating Video: %s (%s)',
video.media.title.text.decode('utf-8'),
video_id)
video_data.title = video.media.title.text.decode(
'utf-8')
video_data.url = video.media.player.url.decode('utf-8')
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode(
'utf-8')
else:
video_data.decription = ' '
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode(
'utf-8')
else:
video_data.keywords = ''
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(
playlist_data.key()):
if video_playlist_key_dict[
playlist_data.key()].has_key(
video_data.key()):
playlist_video = video_playlist_key_dict[
playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(
playlist=playlist_data.key(),
video=video_data.key())
logging.info('Creating VideoPlaylist: %s, %s',
playlist_data.title, video_data.title)
else:
logging.info('Updating VideoPlaylist: %s, %s',
playlist_video.playlist.title,
playlist_video.video.title)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(
video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri=
'http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50'
% playlist_start_index)
def indexVideoData(self):
videos = Video.all().fetch(10000)
for video in videos:
video.index()
video.indexed_title_changed()
def get(self, readable_id=""):
# This method displays a video in the context of a particular topic.
# To do that we first need to find the appropriate topic. If we aren't
# given the topic title in a query param, we need to find a topic that
# the video is a part of. That requires finding the video, given it readable_id
# or, to support old URLs, it's youtube_id.
video = None
topic = None
video_id = self.request.get('v')
topic_id = self.request_string('topic', default="")
readable_id = urllib.unquote(readable_id).decode("utf-8")
readable_id = re.sub(
'-+$', '',
readable_id) # remove any trailing dashes (see issue 1140)
# If either the readable_id or topic title is missing,
# redirect to the canonical URL that contains them
redirect_to_canonical_url = False
if video_id: # Support for old links
query = Video.all()
query.filter('youtube_id =', video_id)
video = query.get()
if not video:
raise MissingVideoException(
"Missing video w/ youtube id '%s'" % video_id)
readable_id = video.readable_id
topic = video.first_topic()
if not topic:
raise MissingVideoException(
"No topic has video w/ youtube id '%s'" % video_id)
redirect_to_canonical_url = True
if topic_id is not None and len(topic_id) > 0:
topic = Topic.get_by_id(topic_id)
key_id = 0 if not topic else topic.key().id()
# If a topic_id wasn't specified or the specified topic wasn't found
# use the first topic for the requested video.
if topic is None:
# Get video by readable_id just to get the first topic for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
topic = video.first_topic()
if not topic:
raise MissingVideoException("No topic has video '%s'" %
readable_id)
redirect_to_canonical_url = True
exid = self.request_string('exid', default=None)
if redirect_to_canonical_url:
qs = {'topic': topic.id}
if exid:
qs['exid'] = exid
urlpath = "/video/%s" % urllib.quote(readable_id)
url = urlparse.urlunparse(
('', '', urlpath, '', urllib.urlencode(qs), ''))
self.redirect(url, True)
return
# If we got here, we have a readable_id and a topic, so we can display
# the topic and the video in it that has the readable_id. Note that we don't
# query the Video entities for one with the requested readable_id because in some
# cases there are multiple Video objects in the datastore with the same readable_id
# (e.g. there are 2 "Order of Operations" videos).
videos = Topic.get_cached_videos_for_topic(topic)
previous_video = None
next_video = None
for v in videos:
if v.readable_id == readable_id:
v.selected = 'selected'
video = v
elif video is None:
previous_video = v
else:
next_video = v
break
# If we're at the beginning or end of a topic, show the adjacent topic.
# previous_topic/next_topic are the topic to display.
# previous_video_topic/next_video_topic are the subtopics the videos
# are actually in.
previous_topic = None
previous_video_topic = None
next_topic = None
next_video_topic = None
if not previous_video:
previous_topic = topic
while not previous_video:
previous_topic = previous_topic.get_previous_topic()
if previous_topic:
(previous_video, previous_video_topic
) = previous_topic.get_last_video_and_topic()
else:
break
if not next_video:
next_topic = topic
while not next_video:
next_topic = next_topic.get_next_topic()
if next_topic:
(next_video, next_video_topic
) = next_topic.get_first_video_and_topic()
else:
break
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
if App.offline_mode:
video_path = "/videos/" + get_mangled_topic_name(
topic.id) + "/" + video.readable_id + ".flv"
else:
video_path = video.download_video_url()
if video.description == video.title:
video.description = None
related_exercises = video.related_exercises()
button_top_exercise = None
if related_exercises:
def ex_to_dict(exercise):
return {
'name': exercise.display_name,
'url': exercise.relative_url,
}
button_top_exercise = ex_to_dict(related_exercises[0])
user_video = UserVideo.get_for_video_and_user_data(
video, UserData.current(), insert_if_missing=True)
awarded_points = 0
if user_video:
awarded_points = user_video.points
subtitles_key_name = VideoSubtitles.get_key_name(
'en', video.youtube_id)
subtitles = VideoSubtitles.get_by_key_name(subtitles_key_name)
subtitles_json = None
if subtitles:
subtitles_json = subtitles.load_json()
template_values = {
'topic': topic,
'video': video,
'videos': videos,
'video_path': video_path,
'video_points_base': consts.VIDEO_POINTS_BASE,
'subtitles_json': subtitles_json,
'button_top_exercise': button_top_exercise,
'related_exercises': [], # disabled for now
'previous_topic': previous_topic,
'previous_video': previous_video,
'previous_video_topic': previous_video_topic,
'next_topic': next_topic,
'next_video': next_video,
'next_video_topic': next_video_topic,
'selected_nav_link': 'watch',
'awarded_points': awarded_points,
'issue_labels': ('Component-Videos,Video-%s' % readable_id),
'author_profile': 'https://plus.google.com/103970106103092409324'
}
template_values = qa.add_template_values(template_values, self.request)
bingo([
'struggling_videos_landing',
'suggested_activity_videos_landing',
'suggested_activity_videos_landing_binary',
])
self.render_jinja2_template('viewvideo.html', template_values)
def get(self):
from exercises import attempt_problem
login_user = UserData.current()
exercises_list = [exercise for exercise in Exercise.all()]
videos_list = [video for video in Video.all()]
user_count = self.request_int('users', 5)
for user_id in xrange(0, user_count):
# Create a new user
first_name = random.choice(CreateRandomGoalData.first_names)
last_name = random.choice(CreateRandomGoalData.last_names)
nickname = "%s %s" % (first_name, last_name)
email = 'test_%i@automatedrandomdata' % user_id
user = users.User(email)
logging.info("Creating user %s: (%i/%i)"
% (nickname, user_id + 1, user_count))
user_data = UserData.get_or_insert(
key_name="test_user_%i" % user_id,
user=user,
current_user=user,
user_id=str(user_id),
moderator=False,
last_login=datetime.now(),
proficient_exercises=[],
suggested_exercises=[],
need_to_reassess=True,
points=0,
coaches=[login_user.user_email],
user_email=email,
user_nickname=nickname,
)
user_data.put()
# Delete user exercise & video progress
query = UserExercise.all()
query.filter('user = '******'user = '******'type': 'GoalObjectiveExerciseProficiency',
'exercise': random.choice(exercises_list)})
for objective in xrange(1, random.randint(2, 4)):
obj_descriptors.append({
'type': 'GoalObjectiveWatchVideo',
'video': random.choice(videos_list)})
title = first_name + "'s Goal #" + str(goal_idx)
logging.info("Creating goal " + title)
objectives = GoalObjective.from_descriptors(obj_descriptors,
user_data)
goal = Goal(parent=user_data, title=title,
objectives=objectives)
user_data.save_goal(goal)
for objective in obj_descriptors:
if objective['type'] == 'GoalObjectiveExerciseProficiency':
user_exercise = user_data.get_or_insert_exercise(
objective['exercise'])
chooser = random.randint(1, 120)
if chooser >= 60:
continue
elif chooser > 15:
count = 1
hints = 0
elif chooser < 7:
count = 20
hints = 0
else:
count = 25
hints = 1
logging.info(
"Starting exercise: %s (%i problems, %i hints)" %
(objective['exercise'].name, count, hints * count))
for i in xrange(1, count):
attempt_problem(user_data, user_exercise, i, 1,
'TEST', 'TEST', 'TEST', True, hints, 0, "TEST",
'TEST', '0.0.0.0')
elif objective['type'] == 'GoalObjectiveWatchVideo':
seconds = random.randint(1, 1200)
logging.info("Watching %i seconds of video %s"
% (seconds, objective['video'].title))
VideoLog.add_entry(user_data, objective['video'],
seconds, 0, detect_cheat=False)
self.response.out.write('OK')
def get(self):
self.response.out.write('<html>')
videos = Video.all()
for video in videos:
self.response.out.write('<P>Title: ' + video.title)
def process_search():
search_query = request.GET.get("search_query", "").strip()
query = search_query.lower()
show_daverank = False
results = False
number_pages = 10
number_videos = 5
# Move this stuff to its own procedure tomorrow!
if query.find("--") == 0:
if query.find("--forum") == 0:
redirect_url = "http://www.udacity-forums.com/cs101/search/?q=" + urllib.quote(query[8:])
return redirect(redirect_url)
if query.find("--cs373") == 0:
redirect_url = "http://www.udacity-forums.com/cs373/search/?q=" + urllib.quote(query[8:])
return redirect(redirect_url)
if query.find("--python") == 0:
redirect_url = "http://docs.python.org/search.html?q=" + urllib.quote(query[9:])
return redirect(redirect_url)
if query.find("--searchwithpeterdotinfo") == 0:
redirect_url = "http://searchwithpeter.info/secretplans.html?q=" + urllib.quote(query[25:])
return redirect(redirect_url)
if query.find("--showmore") == 0:
query = query[11:]
search_query = query
number_pages = 20
number_videos = 10
if query.find("--daverank") == 0:
query = query[11:]
search_query = query
show_daverank = True
if query.find("python") == 0:
pyquery = query[7:]
else:
pyquery = query
ddgurl_root = "http://duckduckgo.com/?q=python+"
ddgurl_suffix = urllib.quote(pyquery) + "&format=json"
response = urllib.urlopen(ddgurl_root + ddgurl_suffix)
response_json = response.read()
pythonterm = json.loads(response_json)
if pythonterm:
pyterm_info = {}
if pythonterm["AbstractSource"] == "Python Documentation":
pyterm = BeautifulSoup(pythonterm["AbstractText"])
try:
pyterm_code = pyterm.find("code").string
pyterm.pre.decompose()
pyterm_info["code"] = pyterm_code
except:
pyterm_info["code"] = None
pyterm_desc = pyterm.get_text()
pyterm_info["desc"] = pyterm_desc
pyterm_info["url"] = pythonterm["AbstractURL"]
results = True
else:
pyterm_info = None
query_words = query.split()
for word in query_words:
if word in stopwords:
query_words.remove(word)
query_urls = []
for term in query_words:
# Get all SearchTerm objects that match the search_query.
q = SearchTerm.all().filter("term =", term).get()
if q:
query_urls.append(set(q.urls))
if query_urls:
query_url_set = set.intersection(*query_urls)
query_url_list = list(query_url_set)
if len(query_url_list) > 0:
results = True
if len(query_url_list) > 30:
query_url_list = query_url_list[0:30]
page_results = Page.all().filter("url IN", query_url_list).order("-dave_rank").fetch(number_pages)
page_dicts = []
for page in page_results:
page_info = {}
query_index = page.text.find(query)
if query_index != -1:
i = page.text.find(" ", query_index - 25)
excerpt_words = page.text[i:].split(" ")
page_info["exact_match"] = True
else:
excerpt_words = page.text.split(" ")
page_info["exact_match"] = False
excerpt = " ".join(excerpt_words[:50])
page_info["text"] = excerpt
page_info["title"] = page.title
page_info["url"] = page.url
page_info["daverank"] = page.dave_rank
page_info["doc"] = page.doc
page_dicts.append(page_info)
page_dicts.sort(key=itemgetter("exact_match"), reverse=True)
video_results = Video.all().filter("url IN", query_url_list).order("-views").fetch(number_videos)
video_dicts = []
for video in video_results:
video_info = {}
subtitles = video.text.lower()
query_index = subtitles.find(query)
time_string = ""
if query_index != -1:
subtitle_list = subtitles.splitlines()
for phrase in subtitle_list:
if phrase.find(query) != -1:
timestamp_index = subtitle_list.index(phrase) - 1
timestamp = subtitle_list[timestamp_index]
if len(timestamp) > 1:
minutes = timestamp[3:5]
seconds = timestamp[6:8]
time_string = "#t=" + minutes + "m" + seconds + "s"
start = 60 * int(minutes) + int(seconds)
if time_string:
url = video.url + time_string
video_info["exact_match"] = True
else:
url = video.url
start = 0
video_info["exact_match"] = False
video_info["title"] = video.title
video_info["url"] = url
video_info["subtitle"] = video.text[-20:query_index:20]
video_info["id"] = video.id
video_info["start"] = start
video_dicts.append(video_info)
video_dicts.sort(key=itemgetter("exact_match"), reverse=True)
else:
page_dicts = None
video_dicts = None
query_string_words = query.split()
return template(
"templates/results",
search_query=search_query,
query_string_words=query_string_words,
page_dicts=page_dicts,
video_dicts=video_dicts,
pyterm_info=pyterm_info,
show_daverank=show_daverank,
results=results,
)
def get(self):
videos = Video.all()
render_template(self, 'index.html', {'videos': videos})
def get(self):
videos = Video.all()
render_template(self, 'index.html', {
'videos': videos
})
def get_video(self, youtube_id):
return Video.all().filter('youtube_id =', youtube_id).get()
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ',
True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(
video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(
playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video": video, "playlist": playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(
fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App': App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html",
**template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def process_search():
search_query = request.GET.get('search_query', '').strip()
query = search_query.lower()
show_daverank = False
results = False
number_pages = 10
number_videos = 5
#Move this stuff to its own procedure tomorrow!
if query.find('--') == 0:
if query.find('--forum') == 0:
redirect_url = 'http://www.udacity-forums.com/cs101/search/?q=' + urllib.quote(
query[8:])
return redirect(redirect_url)
if query.find('--cs373') == 0:
redirect_url = 'http://www.udacity-forums.com/cs373/search/?q=' + urllib.quote(
query[8:])
return redirect(redirect_url)
if query.find('--python') == 0:
redirect_url = 'http://docs.python.org/search.html?q=' + urllib.quote(
query[9:])
return redirect(redirect_url)
if query.find('--searchwithpeterdotinfo') == 0:
redirect_url = 'http://searchwithpeter.info/secretplans.html?q=' + urllib.quote(
query[25:])
return redirect(redirect_url)
if query.find('--showmore') == 0:
query = query[11:]
search_query = query
number_pages = 20
number_videos = 10
if query.find('--daverank') == 0:
query = query[11:]
search_query = query
show_daverank = True
if query.find('python') == 0:
pyquery = query[7:]
else:
pyquery = query
ddgurl_root = 'http://duckduckgo.com/?q=python+'
ddgurl_suffix = urllib.quote(pyquery) + '&format=json'
response = urllib.urlopen(ddgurl_root + ddgurl_suffix)
response_json = response.read()
pythonterm = json.loads(response_json)
if pythonterm:
pyterm_info = {}
if pythonterm['AbstractSource'] == 'Python Documentation':
pyterm = BeautifulSoup(pythonterm['AbstractText'])
try:
pyterm_code = pyterm.find('code').string
pyterm.pre.decompose()
pyterm_info['code'] = pyterm_code
except:
pyterm_info['code'] = None
pyterm_desc = pyterm.get_text()
pyterm_info['desc'] = pyterm_desc
pyterm_info['url'] = pythonterm['AbstractURL']
results = True
else:
pyterm_info = None
query_words = query.split()
for word in query_words:
if word in stopwords:
query_words.remove(word)
query_urls = []
for term in query_words:
# Get all SearchTerm objects that match the search_query.
q = SearchTerm.all().filter('term =', term).get()
if q:
query_urls.append(set(q.urls))
if query_urls:
query_url_set = set.intersection(*query_urls)
query_url_list = list(query_url_set)
if len(query_url_list) > 0:
results = True
if len(query_url_list) > 30:
query_url_list = query_url_list[0:30]
page_results = Page.all().filter(
'url IN', query_url_list).order('-dave_rank').fetch(number_pages)
page_dicts = []
for page in page_results:
page_info = {}
query_index = page.text.find(query)
if query_index != -1:
i = page.text.find(' ', query_index - 25)
excerpt_words = page.text[i:].split(' ')
page_info['exact_match'] = True
else:
excerpt_words = page.text.split(' ')
page_info['exact_match'] = False
excerpt = ' '.join(excerpt_words[:50])
page_info['text'] = excerpt
page_info['title'] = page.title
page_info['url'] = page.url
page_info['daverank'] = page.dave_rank
page_info['doc'] = page.doc
page_dicts.append(page_info)
page_dicts.sort(key=itemgetter('exact_match'), reverse=True)
video_results = Video.all().filter(
'url IN', query_url_list).order('-views').fetch(number_videos)
video_dicts = []
for video in video_results:
video_info = {}
subtitles = video.text.lower()
query_index = subtitles.find(query)
time_string = ''
if query_index != -1:
subtitle_list = subtitles.splitlines()
for phrase in subtitle_list:
if phrase.find(query) != -1:
timestamp_index = subtitle_list.index(phrase) - 1
timestamp = subtitle_list[timestamp_index]
if len(timestamp) > 1:
minutes = timestamp[3:5]
seconds = timestamp[6:8]
time_string = '#t=' + minutes + 'm' + seconds + 's'
start = 60 * int(minutes) + int(seconds)
if time_string:
url = video.url + time_string
video_info['exact_match'] = True
else:
url = video.url
start = 0
video_info['exact_match'] = False
video_info['title'] = video.title
video_info['url'] = url
video_info['subtitle'] = video.text[-20:query_index:20]
video_info['id'] = video.id
video_info['start'] = start
video_dicts.append(video_info)
video_dicts.sort(key=itemgetter('exact_match'), reverse=True)
else:
page_dicts = None
video_dicts = None
query_string_words = query.split()
return template('templates/results',
search_query=search_query,
query_string_words=query_string_words,
page_dicts=page_dicts,
video_dicts=video_dicts,
pyterm_info=pyterm_info,
show_daverank=show_daverank,
results=results)
def get(self, readable_id=""):
# This method displays a video in the context of a particular playlist.
# To do that we first need to find the appropriate playlist. If we aren't
# given the playlist title in a query param, we need to find a playlist that
# the video is a part of. That requires finding the video, given it readable_id
# or, to support old URLs, it's youtube_id.
video = None
playlist = None
video_id = self.request.get('v')
playlist_title = self.request_string('playlist', default="") or self.request_string('p', default="")
readable_id = urllib.unquote(readable_id)
readable_id = re.sub('-+$', '', readable_id) # remove any trailing dashes (see issue 1140)
# If either the readable_id or playlist title is missing,
# redirect to the canonical URL that contains them
redirect_to_canonical_url = False
if video_id: # Support for old links
query = Video.all()
query.filter('youtube_id =', video_id)
video = query.get()
if not video:
raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id)
readable_id = video.readable_id
playlist = video.first_playlist()
if not playlist:
raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id)
redirect_to_canonical_url = True
if playlist_title is not None and len(playlist_title) > 0:
query = Playlist.all().filter('title =', playlist_title)
key_id = 0
for p in query:
if p.key().id() > key_id and not p.youtube_id.endswith('_player'):
playlist = p
key_id = p.key().id()
# If a playlist_title wasn't specified or the specified playlist wasn't found
# use the first playlist for the requested video.
if playlist is None:
# Get video by readable_id just to get the first playlist for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
playlist = video.first_playlist()
if not playlist:
raise MissingVideoException("Missing video '%s'" % readable_id)
redirect_to_canonical_url = True
exid = self.request_string('exid', default=None)
if redirect_to_canonical_url:
qs = {'playlist': playlist.title}
if exid:
qs['exid'] = exid
urlpath = "/video/%s" % urllib.quote(readable_id)
url = urlparse.urlunparse(('', '', urlpath, '', urllib.urlencode(qs), ''))
self.redirect(url, True)
return
# If we got here, we have a readable_id and a playlist_title, so we can display
# the playlist and the video in it that has the readable_id. Note that we don't
# query the Video entities for one with the requested readable_id because in some
# cases there are multiple Video objects in the datastore with the same readable_id
# (e.g. there are 2 "Order of Operations" videos).
videos = VideoPlaylist.get_cached_videos_for_playlist(playlist)
previous_video = None
next_video = None
for v in videos:
if v.readable_id == readable_id:
v.selected = 'selected'
video = v
elif video is None:
previous_video = v
elif next_video is None:
next_video = v
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
if App.offline_mode:
video_path = "/videos/" + get_mangled_playlist_name(playlist_title) + "/" + video.readable_id + ".flv"
else:
video_path = video.download_video_url()
if video.description == video.title:
video.description = None
related_exercises = video.related_exercises()
button_top_exercise = None
if related_exercises:
def ex_to_dict(exercise):
return {
'name': exercise.display_name,
'url': exercise.relative_url,
}
button_top_exercise = ex_to_dict(related_exercises[0])
user_video = UserVideo.get_for_video_and_user_data(video, UserData.current(), insert_if_missing=True)
awarded_points = 0
if user_video:
awarded_points = user_video.points
template_values = {
'playlist': playlist,
'video': video,
'videos': videos,
'video_path': video_path,
'video_points_base': consts.VIDEO_POINTS_BASE,
'button_top_exercise': button_top_exercise,
'related_exercises': [], # disabled for now
'previous_video': previous_video,
'next_video': next_video,
'selected_nav_link': 'watch',
'awarded_points': awarded_points,
'issue_labels': ('Component-Videos,Video-%s' % readable_id),
'author_profile': 'https://plus.google.com/103970106103092409324'
}
template_values = qa.add_template_values(template_values, self.request)
bingo(['struggling_videos_landing',
'homepage_restructure_videos_landing'])
self.render_jinja2_template('viewvideo.html', template_values)
def get(self, readable_id=""):
# This method displays a video in the context of a particular topic.
# To do that we first need to find the appropriate topic. If we aren't
# given the topic title in a query param, we need to find a topic that
# the video is a part of. That requires finding the video, given it readable_id
# or, to support old URLs, it's youtube_id.
video = None
topic = None
video_id = self.request.get('v')
topic_id = self.request_string('topic', default="")
readable_id = urllib.unquote(readable_id).decode("utf-8")
readable_id = re.sub('-+$', '', readable_id) # remove any trailing dashes (see issue 1140)
# If either the readable_id or topic title is missing,
# redirect to the canonical URL that contains them
redirect_to_canonical_url = False
if video_id: # Support for old links
query = Video.all()
query.filter('youtube_id =', video_id)
video = query.get()
if not video:
raise MissingVideoException("Missing video w/ youtube id '%s'" % video_id)
readable_id = video.readable_id
topic = video.first_topic()
if not topic:
raise MissingVideoException("No topic has video w/ youtube id '%s'" % video_id)
redirect_to_canonical_url = True
if topic_id is not None and len(topic_id) > 0:
topic = Topic.get_by_id(topic_id)
key_id = 0 if not topic else topic.key().id()
# If a topic_id wasn't specified or the specified topic wasn't found
# use the first topic for the requested video.
if topic is None:
# Get video by readable_id just to get the first topic for the video
video = Video.get_for_readable_id(readable_id)
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
topic = video.first_topic()
if not topic:
raise MissingVideoException("No topic has video '%s'" % readable_id)
redirect_to_canonical_url = True
exid = self.request_string('exid', default=None)
if redirect_to_canonical_url:
qs = {'topic': topic.id}
if exid:
qs['exid'] = exid
urlpath = "/video/%s" % urllib.quote(readable_id)
url = urlparse.urlunparse(('', '', urlpath, '', urllib.urlencode(qs), ''))
self.redirect(url, True)
return
# If we got here, we have a readable_id and a topic, so we can display
# the topic and the video in it that has the readable_id. Note that we don't
# query the Video entities for one with the requested readable_id because in some
# cases there are multiple Video objects in the datastore with the same readable_id
# (e.g. there are 2 "Order of Operations" videos).
videos = Topic.get_cached_videos_for_topic(topic)
previous_video = None
next_video = None
for v in videos:
if v.readable_id == readable_id:
v.selected = 'selected'
video = v
elif video is None:
previous_video = v
else:
next_video = v
break
# If we're at the beginning or end of a topic, show the adjacent topic.
# previous_topic/next_topic are the topic to display.
# previous_video_topic/next_video_topic are the subtopics the videos
# are actually in.
previous_topic = None
previous_video_topic = None
next_topic = None
next_video_topic = None
if not previous_video:
previous_topic = topic
while not previous_video:
previous_topic = previous_topic.get_previous_topic()
if previous_topic:
(previous_video, previous_video_topic) = previous_topic.get_last_video_and_topic()
else:
break
if not next_video:
next_topic = topic
while not next_video:
next_topic = next_topic.get_next_topic()
if next_topic:
(next_video, next_video_topic) = next_topic.get_first_video_and_topic()
else:
break
if video is None:
raise MissingVideoException("Missing video '%s'" % readable_id)
if App.offline_mode:
video_path = "/videos/" + get_mangled_topic_name(topic.id) + "/" + video.readable_id + ".flv"
else:
video_path = video.download_video_url()
if video.description == video.title:
video.description = None
related_exercises = video.related_exercises()
button_top_exercise = None
if related_exercises:
def ex_to_dict(exercise):
return {
'name': exercise.display_name,
'url': exercise.relative_url,
}
button_top_exercise = ex_to_dict(related_exercises[0])
user_video = UserVideo.get_for_video_and_user_data(video, UserData.current(), insert_if_missing=True)
awarded_points = 0
if user_video:
awarded_points = user_video.points
template_values = {
'topic': topic,
'video': video,
'videos': videos,
'video_path': video_path,
'video_points_base': consts.VIDEO_POINTS_BASE,
'button_top_exercise': button_top_exercise,
'related_exercises': [], # disabled for now
'previous_topic': previous_topic,
'previous_video': previous_video,
'previous_video_topic': previous_video_topic,
'next_topic': next_topic,
'next_video': next_video,
'next_video_topic': next_video_topic,
'selected_nav_link': 'watch',
'awarded_points': awarded_points,
'issue_labels': ('Component-Videos,Video-%s' % readable_id),
'author_profile': 'https://plus.google.com/103970106103092409324',
'is_mobile_allowed': True,
}
template_values = qa.add_template_values(template_values, self.request)
bingo([
'struggling_videos_landing',
'suggested_activity_videos_landing',
'suggested_activity_videos_landing_binary',
])
self.render_jinja2_template('viewvideo.html', template_values)
def updateVideoAndPlaylistData(self):
yt_service = gdata.youtube.service.YouTubeService()
# Now that we run these queries from the App Engine servers, we need to
# explicitly specify our developer_key to avoid being lumped together w/ rest of GAE and
# throttled by YouTube's "Too many request" quota
yt_service.developer_key = "AI39si6ctKTnSR_Vx7o7GpkpeSZAKa6xjbZz6WySzTvKVYRDAO7NHBVwofphk82oP-OSUwIZd0pOJyNuWK8bbOlqzJc9OFozrQ"
yt_service.client_id = "n/a"
video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all())
association_generation = int(Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace('http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/', '')
playlist_uri = playlist.id.text.replace('users/KhanAcademyHebrew/', '')
query = Playlist.all()
query.filter('youtube_id =', playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info('Creating Playlist: %s', playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode("utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(uri=playlist_uri + '?start-index=' + str(start_index) + '&max-results=50')
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media)
continue
video_id = cgi.parse_qs(urlparse(video.media.player.url).query)['v'][0].decode('utf-8')
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info('Found Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info('Creating Video: %s (%s)', video.media.title.text.decode('utf-8'), video_id)
video_data.title = video.media.title.text.decode('utf-8')
video_data.url = video.media.player.url.decode('utf-8')
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode('utf-8')
else:
video_data.decription = ' '
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode('utf-8')
else:
video_data.keywords = ''
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(playlist_data.key()):
if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()):
playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key())
logging.info('Creating VideoPlaylist: %s, %s', playlist_data.title, video_data.title)
else:
logging.info('Updating VideoPlaylist: %s, %s', playlist_video.playlist.title, playlist_video.video.title)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(uri='http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50' % playlist_start_index)
def _task_handler(uid, task_id=0, cursor=None, report=None):
"""Task chain for fetching subtitles from the Universal Subtitles API
It processes Video models in batches of BATCH_SIZE by fetching the English
subtitles via an HTTP API call.
This job runs regularly so fetch failures are fixed from run-to-run. Fetch
failures are logged and suppressed as the task marches on.
Errors include URL fetch timeouts, subtitles put failures, and response
decoding failures.
HTTP redirects indicate that the code needs updating to a new API endpoint.
They are detected and reported separately.
"""
query = Video.all()
query.with_cursor(cursor)
videos = query.fetch(BATCH_SIZE)
if report is None:
report = dict(REPORT_TEMPLATE)
VideoSubtitlesFetchReport(key_name=uid, **report).put()
# Asynchronously fetch. We'll rate-limit by fetching BATCH_SIZE subtitles
# at each DEFER_SECONDS interval
rpcs = []
for video in videos:
url = UNISUBS_URL % urllib.quote(YOUTUBE_URL % video.youtube_id)
rpc = urlfetch.create_rpc(deadline=TIMEOUT_SECONDS)
urlfetch.make_fetch_call(rpc, url)
rpcs.append((video.youtube_id, rpc))
report['fetches'] += 1
# Process asynchronous fetches
for youtube_id, rpc in rpcs:
lang = 'en'
key_name = VideoSubtitles.get_key_name(lang, youtube_id)
try:
resp = rpc.get_result()
if resp.status_code != 200:
raise RuntimeError('status code: %s' % resp.status_code)
if resp.final_url:
logging.warn('%s redirect to %s' % (key_name, resp.final_url))
report['redirects'] += 1
json = resp.content.decode('utf-8')
# Only update stale records
current = VideoSubtitles.get_by_key_name(key_name)
if not current or current.json != json:
new = VideoSubtitles(key_name=key_name,
youtube_id=youtube_id,
language=lang,
json=json)
new.put()
report['writes'] += 1
else:
logging.info('%s content already up-to-date' % key_name)
except Exception, e:
logging.error('%s subtitles fetch failed: %s' % (key_name, e))
report['errors'] += 1
def _task_handler(uid, task_id=0, cursor=None, report=None):
"""Task chain for fetching subtitles from the Universal Subtitles API
It processes Video models in batches of BATCH_SIZE by fetching the English
subtitles via an HTTP API call.
This job runs regularly so fetch failures are fixed from run-to-run. Fetch
failures are logged and suppressed as the task marches on.
Errors include URL fetch timeouts, subtitles put failures, and response
decoding failures.
HTTP redirects indicate that the code needs updating to a new API endpoint.
They are detected and reported separately.
"""
query = Video.all()
query.with_cursor(cursor)
videos = query.fetch(BATCH_SIZE)
if report is None:
report = dict(REPORT_TEMPLATE)
VideoSubtitlesFetchReport(key_name=uid, **report).put()
# Asynchronously fetch. We'll rate-limit by fetching BATCH_SIZE subtitles
# at each DEFER_SECONDS interval
rpcs = []
for video in videos:
url = UNISUBS_URL % urllib.quote(YOUTUBE_URL % video.youtube_id)
rpc = urlfetch.create_rpc(deadline=TIMEOUT_SECONDS)
urlfetch.make_fetch_call(rpc, url)
rpcs.append((video.youtube_id, rpc))
report['fetches'] += 1
# Process asynchronous fetches
for youtube_id, rpc in rpcs:
lang = 'en'
key_name = VideoSubtitles.get_key_name(lang, youtube_id)
try:
resp = rpc.get_result()
if resp.status_code != 200:
raise RuntimeError('status code: %s' % resp.status_code)
if resp.final_url:
logging.warn('%s redirect to %s' % (key_name, resp.final_url))
report['redirects'] += 1
json = resp.content.decode('utf-8')
# Only update stale records
current = VideoSubtitles.get_by_key_name(key_name)
if not current or current.json != json:
new = VideoSubtitles(key_name=key_name, youtube_id=youtube_id,
language=lang, json=json)
new.put()
report['writes'] += 1
else:
logging.info('%s content already up-to-date' % key_name)
except Exception, e:
logging.error('%s subtitles fetch failed: %s' % (key_name, e))
report['errors'] += 1
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ', True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video":video, "playlist":playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App' : App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html", **template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def get_video(self, youtube_id):
return Video.all().filter('youtube_id =', youtube_id).get()
def get(self):
from exercises import attempt_problem
login_user = UserData.current()
exercises_list = [exercise for exercise in Exercise.all()]
videos_list = [video for video in Video.all()]
user_count = self.request_int('users', 5)
for user_id in xrange(0, user_count):
# Create a new user
first_name = random.choice(CreateRandomGoalData.first_names)
last_name = random.choice(CreateRandomGoalData.last_names)
nickname = "%s %s" % (first_name, last_name)
email = 'test_%i@automatedrandomdata' % user_id
user = users.User(email)
logging.info("Creating user %s: (%i/%i)"
% (nickname, user_id + 1, user_count))
user_data = UserData.get_or_insert(
key_name="test_user_%i" % user_id,
user=user,
current_user=user,
user_id=str(user_id),
moderator=False,
last_login=datetime.now(),
proficient_exercises=[],
suggested_exercises=[],
need_to_reassess=True,
points=0,
coaches=[login_user.user_email],
user_email=email,
user_nickname=nickname,
)
user_data.put()
# Delete user exercise & video progress
query = UserExercise.all()
query.filter('user = '******'user = '******'type': 'GoalObjectiveExerciseProficiency',
'exercise': random.choice(exercises_list)})
for objective in xrange(1, random.randint(2, 4)):
obj_descriptors.append({
'type': 'GoalObjectiveWatchVideo',
'video': random.choice(videos_list)})
title = first_name + "'s Goal #" + str(goal_idx)
logging.info("Creating goal " + title)
objectives = GoalObjective.from_descriptors(obj_descriptors,
user_data)
goal = Goal(parent=user_data, title=title,
objectives=objectives)
user_data.save_goal(goal)
for objective in obj_descriptors:
if objective['type'] == 'GoalObjectiveExerciseProficiency':
user_exercise = user_data.get_or_insert_exercise(
objective['exercise'])
chooser = random.randint(1, 120)
if chooser >= 60:
continue
elif chooser > 15:
count = 1
hints = 0
elif chooser < 7:
count = 20
hints = 0
else:
count = 25
hints = 1
logging.info(
"Starting exercise: %s (%i problems, %i hints)" %
(objective['exercise'].name, count, hints * count))
for i in xrange(1, count):
attempt_problem(user_data, user_exercise, i, 1,
'TEST', 'TEST', 'TEST', True, hints, 0, False,
"TEST", 'TEST', '0.0.0.0')
elif objective['type'] == 'GoalObjectiveWatchVideo':
seconds = random.randint(1, 1200)
logging.info("Watching %i seconds of video %s"
% (seconds, objective['video'].title))
VideoLog.add_entry(user_data, objective['video'],
seconds, 0, detect_cheat=False)
self.response.out.write('OK')
def updateVideoAndPlaylistData(self):
yt_service = YouTubeService()
video_youtube_id_dict = Video.get_dict(Video.all(), lambda video: video.youtube_id)
video_playlist_key_dict = VideoPlaylist.get_key_dict(VideoPlaylist.all())
association_generation = int(Setting.last_youtube_sync_generation_start())
logging.info("Fetching playlists")
playlist_start_index = 1
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50"
% playlist_start_index
)
while len(playlist_feed.entry) > 0:
for playlist in playlist_feed.entry:
logging.info("Playlist: %s", playlist.id.text)
playlist_id = playlist.id.text.replace(
"http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists/", ""
)
playlist_uri = playlist.id.text.replace("users/KhanAcademyHebrew/", "")
query = Playlist.all()
query.filter("youtube_id =", playlist_id)
playlist_data = query.get()
if not playlist_data:
playlist_data = Playlist(youtube_id=playlist_id)
logging.info("Creating Playlist: %s", playlist.title.text)
playlist_data.url = playlist_uri
playlist_data.title = playlist.title.text.decode("utf-8")
playlist_data.description = playlist.description.text.decode("utf-8")
playlist_data.tags = []
for category in playlist.category:
if "tags.cat" in category.scheme:
playlist_data.tags.append(category.term)
playlist_data.put()
for i in range(0, 10):
start_index = i * 50 + 1
video_feed = yt_service.GetYouTubePlaylistVideoFeed(
uri=playlist_uri + "?start-index=" + str(start_index) + "&max-results=50"
)
video_data_list = []
if len(video_feed.entry) <= 0:
# No more videos in playlist
break
for video in video_feed.entry:
if not video.media.player:
logging.warning("Could not parse video - skipping... (%s, %s)", video, video.media)
continue
video_id = cgi.parse_qs(urlparse(video.media.player.url).query)["v"][0].decode("utf-8")
video_data = None
if video_youtube_id_dict.has_key(video_id):
video_data = video_youtube_id_dict[video_id]
logging.info("Found Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id)
if not video_data:
video_data = Video(youtube_id=video_id)
logging.info("Creating Video: %s (%s)", video.media.title.text.decode("utf-8"), video_id)
video_data.title = video.media.title.text.decode("utf-8")
video_data.url = video.media.player.url.decode("utf-8")
video_data.duration = int(video.media.duration.seconds)
if video.statistics:
video_data.views = int(video.statistics.view_count)
if video.media.description.text is not None:
video_data.description = video.media.description.text.decode("utf-8")
else:
video_data.decription = " "
if video.media.keywords.text:
video_data.keywords = video.media.keywords.text.decode("utf-8")
else:
video_data.keywords = ""
video_data.position = video.position
video_data_list.append(video_data)
db.put(video_data_list)
playlist_videos = []
for video_data in video_data_list:
playlist_video = None
if video_playlist_key_dict.has_key(playlist_data.key()):
if video_playlist_key_dict[playlist_data.key()].has_key(video_data.key()):
playlist_video = video_playlist_key_dict[playlist_data.key()][video_data.key()]
if not playlist_video:
playlist_video = VideoPlaylist(playlist=playlist_data.key(), video=video_data.key())
logging.info("Creating VideoPlaylist: %s, %s", playlist_data.title, video_data.title)
else:
logging.info(
"Updating VideoPlaylist: %s, %s",
playlist_video.playlist.title,
playlist_video.video.title,
)
playlist_video.last_live_association_generation = association_generation
playlist_video.video_position = int(video_data.position.text)
playlist_videos.append(playlist_video)
db.put(playlist_videos)
# Check next set of playlists
playlist_start_index += 50
playlist_feed = yt_service.GetYouTubePlaylistFeed(
uri="http://gdata.youtube.com/feeds/api/users/KhanAcademyHebrew/playlists?start-index=%s&max-results=50"
% playlist_start_index
)
