def get_modified_between_dts(user_data, dt_a, dt_b):
query_updated = GoalList.get_updated_between_dts(user_data, dt_a, dt_b)
query_created = GoalList.get_created_between_dts(user_data, dt_a, dt_b)
results = util.async_queries([query_updated, query_created], limit=200)
goals = set(results[0].get_result()) or set(results[1].get_result())
return list(goals)
def fill_class_summaries_from_logs(user_data_coach, students_data, dt_start_utc):
dt_end_utc = dt_start_utc + datetime.timedelta(days = 1)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(user_data_student, dt_start_utc, dt_end_utc)
query_video_logs = VideoLog.get_for_user_data_between_dts(user_data_student, dt_start_utc, dt_end_utc)
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
logging.info("working on student "+str(user_data_student.user))
problem_and_video_logs = []
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs, key=lambda log: log.time_started())
if problem_and_video_logs:
LogSummary.add_or_update_entry(user_data_coach, problem_and_video_logs, ClassDailyActivitySummary, LogSummaryTypes.CLASS_DAILY_ACTIVITY, 1440)
def get_modified_between_dts(user_data, dt_a, dt_b):
query_updated = GoalList.get_updated_between_dts(user_data, dt_a, dt_b)
query_created = GoalList.get_created_between_dts(user_data, dt_a, dt_b)
results = util.async_queries([query_updated, query_created], limit=200)
goals = set(results[0].get_result()) or set(results[1].get_result())
return list(goals)
def fill_realtime_recent_daily_activity_summaries(daily_activity_logs, user_data, dt_end):
if user_data.last_daily_summary and dt_end <= user_data.last_daily_summary:
return daily_activity_logs
# We're willing to fill the last 4 days with realtime data if summary logs haven't
# been compiled for some reason.
dt_end = min(dt_end, datetime.datetime.now())
dt_start = dt_end - datetime.timedelta(days=4)
if user_data.last_daily_summary:
dt_start = max(dt_start, user_data.last_daily_summary)
query_problem_logs = models.ProblemLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
query_video_logs = models.VideoLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
results = util.async_queries([query_problem_logs, query_video_logs])
problem_logs = results[0].get_result()
video_logs = results[1].get_result()
# Chop off hours, minutes, and seconds
dt_start = datetime.datetime(dt_start.year, dt_start.month, dt_start.day)
dt_end = datetime.datetime(dt_end.year, dt_end.month, dt_end.day)
dt = dt_start
while dt <= dt_end:
summary = DailyActivitySummary.build(user_data, dt, problem_logs, video_logs)
if summary.has_activity():
log = models.DailyActivityLog.build(user_data, dt, summary)
daily_activity_logs.append(log)
dt += datetime.timedelta(days=1)
return daily_activity_logs
def fill_realtime_recent_daily_activity_summaries(daily_activity_logs, user_data, dt_end):
if user_data.last_daily_summary and dt_end <= user_data.last_daily_summary:
return daily_activity_logs
# We're willing to fill the last 4 days with realtime data if summary logs haven't
# been compiled for some reason.
dt_end = min(dt_end, datetime.datetime.now())
dt_start = dt_end - datetime.timedelta(days=4)
if user_data.last_daily_summary:
dt_start = max(dt_start, user_data.last_daily_summary)
query_problem_logs = models.ProblemLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
query_video_logs = models.VideoLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
results = util.async_queries([query_problem_logs, query_video_logs])
problem_logs = results[0].get_result()
video_logs = results[1].get_result()
# Chop off hours, minutes, and seconds
dt_start = datetime.datetime(dt_start.year, dt_start.month, dt_start.day)
dt_end = datetime.datetime(dt_end.year, dt_end.month, dt_end.day)
dt = dt_start
while dt <= dt_end:
summary = DailyActivitySummary.build(user_data, dt, problem_logs, video_logs)
if summary.has_activity():
log = models.DailyActivityLog.build(user_data, dt, summary)
daily_activity_logs.append(log)
dt += datetime.timedelta(days=1)
return daily_activity_logs
def class_exercises_over_time_graph_context(user_data, student_list):
if not user_data:
return {}
end_date = None
if student_list:
students_data = student_list.get_students_data()
else:
students_data = user_data.get_students_data()
dict_student_exercises = {}
dict_exercises = {}
async_queries = []
for user_data_student in students_data:
query = models.UserExercise.all()
query.filter('user ='******'proficient_date >', None)
query.order('proficient_date')
async_queries.append(query)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
student_nickname = user_data_student.nickname
dict_student_exercises[student_nickname] = {
"nickname": student_nickname,
"email": user_data_student.email,
"exercises": []
}
exercises = results[i].get_result()
for user_exercise in exercises:
joined = min(user_data.joined, user_exercise.proficient_date)
days_until_proficient = (user_exercise.proficient_date -
joined).days
proficient_date = user_exercise.proficient_date.strftime(
'%m/%d/%Y')
data = ExerciseData(student_nickname, user_exercise.exercise,
user_exercise.exercise, days_until_proficient,
proficient_date)
dict_student_exercises[student_nickname]["exercises"].append(data)
end_date = user_exercise.proficient_date
return {
"dict_student_exercises":
dict_student_exercises,
"user_data_students":
students_data,
"c_points":
reduce(lambda a, b: a + b, map(lambda s: s.points, students_data), 0)
}
def recent_activity_for(user_data, dt_start, dt_end):
query_user_badges = models_badges.UserBadge.get_for_user_data_between_dts(user_data, dt_start, dt_end)
query_problem_logs = models.ProblemLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
query_video_logs = models.VideoLog.get_for_user_data_between_dts(user_data, dt_start, dt_end)
results = util.async_queries([query_user_badges, query_problem_logs, query_video_logs], limit=200)
list_recent_activity_types = [
recent_badge_activity(results[0].get_result()),
recent_exercise_activity(results[1].get_result()),
recent_video_activity(results[2].get_result()),
]
list_recent_activity = [activity for sublist in list_recent_activity_types for activity in sublist]
return collapse_recent_activity(list_recent_activity)[:MOST_RECENT_ITEMS]
def class_exercises_over_time_graph_context(user_data, student_list):
if not user_data:
return {}
end_date = None
if student_list:
students_data = student_list.get_students_data()
else:
students_data = user_data.get_students_data()
dict_student_exercises = {}
dict_exercises = {}
async_queries = []
for user_data_student in students_data:
query = models.UserExercise.all()
query.filter('user ='******'proficient_date >', None)
query.order('proficient_date')
async_queries.append(query)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
student_nickname = user_data_student.nickname
dict_student_exercises[student_nickname] = { "nickname": student_nickname, "email": user_data_student.email, "exercises": [] }
exercises = results[i].get_result()
for user_exercise in exercises:
joined = min(user_data.joined, user_exercise.proficient_date)
days_until_proficient = (user_exercise.proficient_date - joined).days
proficient_date = user_exercise.proficient_date.strftime('%m/%d/%Y')
data = ExerciseData(student_nickname, user_exercise.exercise, user_exercise.exercise, days_until_proficient, proficient_date)
dict_student_exercises[student_nickname]["exercises"].append(data)
end_date = user_exercise.proficient_date
return {
"dict_student_exercises": dict_student_exercises,
"user_data_students": students_data,
"c_points": reduce(lambda a, b: a + b, map(lambda s: s.points, students_data), 0)
}
def fill_class_summaries_from_logs(user_data_coach, students_data,
dt_start_utc):
dt_end_utc = dt_start_utc + datetime.timedelta(days=1)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(
user_data_student, dt_start_utc, dt_end_utc)
query_video_logs = VideoLog.get_for_user_data_between_dts(
user_data_student, dt_start_utc, dt_end_utc)
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
logging.info("working on student " + str(user_data_student.user))
problem_and_video_logs = []
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs,
key=lambda log: log.time_started())
if problem_and_video_logs:
LogSummary.add_or_update_entry(
user_data_coach, problem_and_video_logs,
ClassDailyActivitySummary,
LogSummaryTypes.CLASS_DAILY_ACTIVITY, 1440)
def recent_activity_for(user_data, dt_start, dt_end):
query_user_badges = models_badges.UserBadge.get_for_user_data_between_dts(
user_data, dt_start, dt_end)
query_problem_logs = models.ProblemLog.get_for_user_data_between_dts(
user_data, dt_start, dt_end)
query_video_logs = models.VideoLog.get_for_user_data_between_dts(
user_data, dt_start, dt_end)
results = util.async_queries(
[query_user_badges, query_problem_logs, query_video_logs], limit=200)
list_recent_activity_types = [
recent_badge_activity(results[0].get_result()),
recent_exercise_activity(results[1].get_result()),
recent_video_activity(results[2].get_result()),
]
list_recent_activity = [
activity for sublist in list_recent_activity_types
for activity in sublist
]
return collapse_recent_activity(list_recent_activity)[:MOST_RECENT_ITEMS]
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ',
True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(
video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(
playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video": video, "playlist": playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(
fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App': App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html",
**template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def library_content_html():
# No cache found -- regenerate HTML
smart_history = getSmartHistoryContent()
all_playlists = []
dict_videos = {}
dict_videos_counted = {}
dict_playlists = {}
dict_playlists_by_title = {}
dict_video_playlists = {}
async_queries = [
Video.all(),
Playlist.all(),
VideoPlaylist.all().filter('live_association = ', True).order('video_position'),
]
results = util.async_queries(async_queries)
for video in results[0].get_result():
dict_videos[video.key()] = video
for playlist in results[1].get_result():
dict_playlists[playlist.key()] = playlist
if playlist.title in topics_list:
dict_playlists_by_title[playlist.title] = playlist
for video_playlist in results[2].get_result():
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(video_playlist)
video_key = VideoPlaylist.video.get_value_for_datastore(video_playlist)
if dict_videos.has_key(video_key) and dict_playlists.has_key(playlist_key):
video = dict_videos[video_key]
playlist = dict_playlists[playlist_key]
fast_video_playlist_dict = {"video":video, "playlist":playlist}
if dict_video_playlists.has_key(playlist_key):
dict_video_playlists[playlist_key].append(fast_video_playlist_dict)
else:
dict_video_playlists[playlist_key] = [fast_video_playlist_dict]
if dict_playlists_by_title.has_key(playlist.title):
# Only count videos in topics_list
dict_videos_counted[video.youtube_id] = True
# Update count of all distinct videos associated w/ a live playlist
Setting.count_videos(len(dict_videos_counted.keys()))
for topic in topics_list:
if topic in dict_playlists_by_title:
playlist = dict_playlists_by_title[topic]
playlist_key = playlist.key()
playlist_videos = dict_video_playlists.get(playlist_key) or []
if not playlist_videos:
logging.error('Playlist %s has no videos!', playlist.title)
playlist_data = {
'title': topic,
'topic': topic,
'playlist': playlist,
'videos': playlist_videos,
'next': None
}
all_playlists.append(playlist_data)
playlist_data_prev = None
for playlist_data in all_playlists:
if playlist_data_prev:
playlist_data_prev['next'] = playlist_data
playlist_data_prev = playlist_data
# Separating out the columns because the formatting is a little different on each column
template_values = {
'App' : App,
'all_playlists': all_playlists,
'smart_history': smart_history,
}
html = shared_jinja.get().render_template("library_content_template.html", **template_values)
# Set shared date of last generated content
Setting.cached_library_content_date(str(datetime.datetime.now()))
return html
def get(self):
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update({
'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH
})
self.render_jinja2_template("searchresults.html", template_values)
return
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query, limit=50, kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_key_list = [str(key_and_title[0]) for key_and_title in all_text_keys]
# all_key_list.extend([result["key"] for result in topic_partial_results])
all_key_list.extend([result["key"] for result in video_partial_results])
all_key_list.extend([result["key"] for result in url_partial_results])
all_key_list = list(set(all_key_list))
# Filter out anything that isn't a Topic, Url or Video
all_key_list = [key for key in all_key_list if db.Key(key).kind() in ["Topic", "Url", "Video"]]
# Get all the entities
all_entities = db.get(all_key_list)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
topic_count = len(topics)
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys) for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(lambda exkey: [exercise for exercise in exercises if exercise.key() == exkey][0], exercise_keys)
# Count number of videos in each topic and sort descending
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [(str(topic.key()) in video.topic_string_keys) for video in filtered_videos].count(True)
topics = sorted(topics, key=lambda topic: -topic.match_count)
else:
for topic in topics:
topic.match_count = 0
template_values.update({
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
})
self.render_jinja2_template("searchresults.html", template_values)
def get_old(self):
""" Deprecated old version of search, so we can Gandalf in the new one.
If new search is working, this should be taken out by May 31, 2012.
"""
show_update = False
if App.is_dev_server and user_util.is_current_user_admin():
update = self.request_bool("update", False)
if update:
self.update()
version_number = layer_cache.KeyValueCache.get(
"last_dev_topic_vesion_indexed")
default_version = topic_models.TopicVersion.get_default_version()
if version_number != default_version.number:
show_update = True
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update({
'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH
})
self.render_jinja2_template("searchresults.html", template_values)
return
searched_phrases = []
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query, limit=50, kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD,
searched_phrases_out=searched_phrases)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_keys = [str(key_and_title[0]) for key_and_title in all_text_keys]
all_keys.extend([result["key"] for result in topic_partial_results])
all_keys.extend([result["key"] for result in video_partial_results])
all_keys.extend([result["key"] for result in url_partial_results])
all_keys = list(set(all_keys))
# Filter out anything that isn't a Topic, Url or Video
all_keys = [key for key in all_keys
if db.Key(key).kind() in ["Topic", "Url", "Video"]]
# Get all the entities
all_entities = db.get(all_keys)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys)
for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(
exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(
lambda exkey: [exercise for exercise in exercises
if exercise.key() == exkey][0], exercise_keys)
# Count number of videos in each topic and sort descending
topic_count = 0
matching_topic_count = 0
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [
(str(topic.key()) in video.topic_string_keys)
for video in filtered_videos].count(True)
if topic.match_count > 0:
topic_count += 1
topics = sorted(topics,
key=lambda topic: topic.match_count,
reverse=True)
else:
for topic in topics:
topic.match_count = 0
for topic in topics:
if topic.title.lower() == query:
topic.matches = True
matching_topic_count += 1
child_topics = topic.get_child_topics(
include_descendants=True)
topic.child_topics = [t for t in child_topics
if t.has_content()]
template_values.update({
'show_update': show_update,
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
'matching_topic_count': matching_topic_count
})
self.render_jinja2_template("searchresults.html", template_values)
def get_classtime_table_old(self, students_data, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days = 1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = exercise_models.ProblemLog.get_for_user_data_between_dts(user_data_student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
query_video_logs = video_models.VideoLog.get_for_user_data_between_dts(user_data_student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
rows = 0
chunks = 0
for i, user_data_student in enumerate(students_data):
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs, key=lambda log: log.time_started())
rows += len(problem_and_video_logs)
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(activity.time_started()) > (chunk_current.end + self.chunk_delta):
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.user_data_student = user_data_student
chunk_current.start = self.dt_to_ctz(activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()), dt_end_ctz)
if chunk_current is not None:
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
column += 1
logging.info("old rows="+str(rows)+", old chunks="+str(chunks))
classtime_table.balance()
return classtime_table
def get(self):
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update({
'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH
})
self.render_jinja2_template("searchresults.html", template_values)
return
searched_phrases = []
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Playlist.full_text_search(
query, limit=50, kind=None,
stemming=Playlist.INDEX_STEMMING,
multi_word_literal=Playlist.INDEX_MULTI_WORD,
searched_phrases_out=searched_phrases)
# Quick title-only partial search
playlist_partial_results = filter(
lambda playlist_dict: query in playlist_dict["title"].lower(),
autocomplete.playlist_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
# Combine results & do one big get!
all_key_list = [str(key_and_title[0]) for key_and_title in all_text_keys]
#all_key_list.extend([result["key"] for result in playlist_partial_results])
all_key_list.extend([result["key"] for result in video_partial_results])
all_key_list = list(set(all_key_list))
all_entities = db.get(all_key_list)
# Filter results by type
playlists = []
videos = []
for entity in all_entities:
if isinstance(entity, Playlist):
playlists.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif entity is not None:
logging.error("Unhandled kind in search results: " +
str(type(entity)))
playlist_count = len(playlists)
# Get playlists for videos not in matching playlists
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(playlist.title in video.playlists) for playlist in playlists].count(True) == 0:
video_playlist = video.first_playlist()
if video_playlist != None:
playlists.append(video_playlist)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(lambda exkey: [exercise for exercise in exercises if exercise.key() == exkey][0], exercise_keys)
# Count number of videos in each playlist and sort descending
for playlist in playlists:
if len(filtered_videos) > 0:
playlist.match_count = [(playlist.title in video.playlists) for video in filtered_videos].count(True)
else:
playlist.match_count = 0
playlists = sorted(playlists, key=lambda playlist: -playlist.match_count)
template_values.update({
'playlists': playlists,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'playlist_count': playlist_count,
})
self.render_jinja2_template("searchresults.html", template_values)
def get_classtime_table_by_coach(self, user_data_coach, students_data,
dt_start_utc):
logging.info("getting classtime table for " + str(dt_start_utc))
#ctz will be from midnight to midnight on the day they are looking at
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days=1)
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# midnight at PST is 7AM UTC and hence the coach's day in UTC goes from 7AM to 7AM the next day, spanning two different UTC days
dt_end_utc = dt_start_utc + datetime.timedelta(days=1)
# find the first utc days that spans the teacher's day
dt_start_utc1 = datetime.datetime(dt_start_utc.year,
dt_start_utc.month, dt_start_utc.day)
dt_end_utc1 = dt_start_utc1 + datetime.timedelta(days=1)
# get the query to get the summary shards from the first day
log_summary_query_1 = LogSummary.get_by_name(
LogSummary.get_name_by_dates(user_data_coach,
LogSummaryTypes.CLASS_DAILY_ACTIVITY,
dt_start_utc1, dt_end_utc1))
# find the second utc day that spans the teacher's day
dt_start_utc2 = dt_end_utc1
dt_end_utc2 = dt_start_utc2 + datetime.timedelta(days=1)
log_summary_query_2 = LogSummary.get_by_name(
LogSummary.get_name_by_dates(user_data_coach,
LogSummaryTypes.CLASS_DAILY_ACTIVITY,
dt_start_utc2, dt_end_utc2))
results = util.async_queries(
[log_summary_query_1, log_summary_query_2], limit=10000)
class_summary_shards = results[0].get_result()
class_summary = None
if class_summary_shards:
class_summary = reduce(
lambda x, y: x.merge_shard(y),
map(lambda x: x.summary, class_summary_shards))
class_summary_day2_shards = results[1].get_result()
class_summary_day2 = None
if class_summary_day2_shards:
class_summary_day2 = reduce(
lambda x, y: x.merge_shard(y),
map(lambda x: x.summary, class_summary_day2_shards))
if class_summary_day2 is not None:
if class_summary is not None:
class_summary.merge_day(class_summary_day2)
else:
class_summary = class_summary_day2
if not class_summary:
return classtime_table
rows = 0
# only consider sudents that are in the coach's currently looked at list (some students might have stopped having their current coach, or we might only be interested in a coach's student_list
for i, user_data_student in enumerate(students_data):
# check to see if the current student has had any activity
if class_summary.student_dict.has_key(user_data_student.user):
# loop over all chunks of that day
for adjacent_activity_summary in class_summary.student_dict[
user_data_student.user]:
# make sure the chunk falls within the day specified by the coach's timezone
if adjacent_activity_summary.start > dt_start_utc and adjacent_activity_summary.start < dt_end_utc:
rows += 1
adjacent_activity_summary.setTimezoneOffset(
self.timezone_offset)
classtime_table.drop_into_column(
adjacent_activity_summary, i)
logging.info("summary by coach rows=" + str(rows))
return classtime_table
def get(self):
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update(
{'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH})
self.render_jinja2_template("searchresults.html", template_values)
return
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query,
limit=50,
kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_key_list = [
str(key_and_title[0]) for key_and_title in all_text_keys
]
# all_key_list.extend([result["key"] for result in topic_partial_results])
all_key_list.extend(
[result["key"] for result in video_partial_results])
all_key_list.extend([result["key"] for result in url_partial_results])
all_key_list = list(set(all_key_list))
# Filter out anything that isn't a Topic, Url or Video
all_key_list = [
key for key in all_key_list
if db.Key(key).kind() in ["Topic", "Url", "Video"]
]
# Get all the entities
all_entities = db.get(all_key_list)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
topic_count = len(topics)
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys)
for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(
exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(
lambda exkey: [
exercise for exercise in exercises
if exercise.key() == exkey
][0], exercise_keys)
# Count number of videos in each topic and sort descending
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [
(str(topic.key()) in video.topic_string_keys)
for video in filtered_videos
].count(True)
topics = sorted(topics, key=lambda topic: -topic.match_count)
else:
for topic in topics:
topic.match_count = 0
template_values.update({
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
})
self.render_jinja2_template("searchresults.html", template_values)
def get_classtime_table_by_coach(self, user_data_coach, students_data, dt_start_utc):
logging.info("getting classtime table for "+str(dt_start_utc))
#ctz will be from midnight to midnight on the day they are looking at
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days = 1)
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# midnight at PST is 7AM UTC and hence the coach's day in UTC goes from 7AM to 7AM the next day, spanning two different UTC days
dt_end_utc = dt_start_utc + datetime.timedelta(days = 1)
# find the first utc days that spans the teacher's day
dt_start_utc1 = datetime.datetime(dt_start_utc.year, dt_start_utc.month, dt_start_utc.day)
dt_end_utc1 = dt_start_utc1 + datetime.timedelta(days = 1)
# get the query to get the summary shards from the first day
log_summary_query_1 = LogSummary.get_by_name(LogSummary.get_name_by_dates(user_data_coach, LogSummaryTypes.CLASS_DAILY_ACTIVITY, dt_start_utc1, dt_end_utc1))
# find the second utc day that spans the teacher's day
dt_start_utc2 = dt_end_utc1
dt_end_utc2 = dt_start_utc2 + datetime.timedelta(days = 1)
log_summary_query_2 = LogSummary.get_by_name(LogSummary.get_name_by_dates(user_data_coach, LogSummaryTypes.CLASS_DAILY_ACTIVITY, dt_start_utc2, dt_end_utc2))
results = util.async_queries([log_summary_query_1, log_summary_query_2], limit = 10000)
class_summary_shards = results[0].get_result()
class_summary = None
if class_summary_shards:
class_summary = reduce(lambda x, y: x.merge_shard(y), map(lambda x: x.summary, class_summary_shards))
class_summary_day2_shards = results[1].get_result()
class_summary_day2 = None
if class_summary_day2_shards:
class_summary_day2 = reduce(lambda x, y: x.merge_shard(y), map(lambda x: x.summary, class_summary_day2_shards))
if class_summary_day2 is not None:
if class_summary is not None :
class_summary.merge_day(class_summary_day2)
else:
class_summary = class_summary_day2
if not class_summary:
return classtime_table
rows = 0
# only consider sudents that are in the coach's currently looked at list (some students might have stopped having their current coach, or we might only be interested in a coach's student_list
for i, user_data_student in enumerate(students_data):
# check to see if the current student has had any activity
if class_summary.student_dict.has_key(user_data_student.user):
# loop over all chunks of that day
for adjacent_activity_summary in class_summary.student_dict[user_data_student.user]:
# make sure the chunk falls within the day specified by the coach's timezone
if adjacent_activity_summary.start > dt_start_utc and adjacent_activity_summary.start < dt_end_utc:
rows += 1
adjacent_activity_summary.setTimezoneOffset(self.timezone_offset)
classtime_table.drop_into_column(adjacent_activity_summary, i)
logging.info("summary by coach rows="+str(rows))
return classtime_table
def get_old(self):
""" Deprecated old version of search, so we can Gandalf in the new one.
If new search is working, this should be taken out by May 31, 2012.
"""
show_update = False
if App.is_dev_server and user_util.is_current_user_admin():
update = self.request_bool("update", False)
if update:
self.update()
version_number = layer_cache.KeyValueCache.get(
"last_dev_topic_vesion_indexed")
default_version = topic_models.TopicVersion.get_default_version()
if version_number != default_version.number:
show_update = True
query = self.request.get('page_search_query')
template_values = {'page_search_query': query}
query = query.strip()
if len(query) < search.SEARCH_PHRASE_MIN_LENGTH:
if len(query) > 0:
template_values.update(
{'query_too_short': search.SEARCH_PHRASE_MIN_LENGTH})
self.render_jinja2_template("searchresults.html", template_values)
return
searched_phrases = []
# Do an async query for all ExerciseVideos, since this may be slow
exvids_query = ExerciseVideo.all()
exvids_future = util.async_queries([exvids_query])
# One full (non-partial) search, then sort by kind
all_text_keys = Topic.full_text_search(
query,
limit=50,
kind=None,
stemming=Topic.INDEX_STEMMING,
multi_word_literal=Topic.INDEX_MULTI_WORD,
searched_phrases_out=searched_phrases)
# Quick title-only partial search
topic_partial_results = filter(
lambda topic_dict: query in topic_dict["title"].lower(),
autocomplete.topic_title_dicts())
video_partial_results = filter(
lambda video_dict: query in video_dict["title"].lower(),
autocomplete.video_title_dicts())
url_partial_results = filter(
lambda url_dict: query in url_dict["title"].lower(),
autocomplete.url_title_dicts())
# Combine results & do one big get!
all_keys = [str(key_and_title[0]) for key_and_title in all_text_keys]
all_keys.extend([result["key"] for result in topic_partial_results])
all_keys.extend([result["key"] for result in video_partial_results])
all_keys.extend([result["key"] for result in url_partial_results])
all_keys = list(set(all_keys))
# Filter out anything that isn't a Topic, Url or Video
all_keys = [
key for key in all_keys
if db.Key(key).kind() in ["Topic", "Url", "Video"]
]
# Get all the entities
all_entities = db.get(all_keys)
# Group results by type
topics = []
videos = []
for entity in all_entities:
if isinstance(entity, Topic):
topics.append(entity)
elif isinstance(entity, Video):
videos.append(entity)
elif isinstance(entity, Url):
videos.append(entity)
elif entity:
logging.info("Found unknown object " + repr(entity))
# Get topics for videos not in matching topics
filtered_videos = []
filtered_videos_by_key = {}
for video in videos:
if [(str(topic.key()) in video.topic_string_keys)
for topic in topics].count(True) == 0:
video_topic = video.first_topic()
if video_topic != None:
topics.append(video_topic)
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
else:
filtered_videos.append(video)
filtered_videos_by_key[str(video.key())] = []
video_count = len(filtered_videos)
# Get the related exercises
all_exercise_videos = exvids_future[0].get_result()
exercise_keys = []
for exvid in all_exercise_videos:
video_key = str(ExerciseVideo.video.get_value_for_datastore(exvid))
if video_key in filtered_videos_by_key:
exercise_key = ExerciseVideo.exercise.get_value_for_datastore(
exvid)
video_exercise_keys = filtered_videos_by_key[video_key]
video_exercise_keys.append(exercise_key)
exercise_keys.append(exercise_key)
exercises = db.get(exercise_keys)
# Sort exercises with videos
video_exercises = {}
for video_key, exercise_keys in filtered_videos_by_key.iteritems():
video_exercises[video_key] = map(
lambda exkey: [
exercise for exercise in exercises
if exercise.key() == exkey
][0], exercise_keys)
# Count number of videos in each topic and sort descending
topic_count = 0
matching_topic_count = 0
if topics:
if len(filtered_videos) > 0:
for topic in topics:
topic.match_count = [
(str(topic.key()) in video.topic_string_keys)
for video in filtered_videos
].count(True)
if topic.match_count > 0:
topic_count += 1
topics = sorted(topics,
key=lambda topic: topic.match_count,
reverse=True)
else:
for topic in topics:
topic.match_count = 0
for topic in topics:
if topic.title.lower() == query:
topic.matches = True
matching_topic_count += 1
child_topics = topic.get_child_topics(
include_descendants=True)
topic.child_topics = [
t for t in child_topics if t.has_content()
]
template_values.update({
'show_update': show_update,
'topics': topics,
'videos': filtered_videos,
'video_exercises': video_exercises,
'search_string': query,
'video_count': video_count,
'topic_count': topic_count,
'matching_topic_count': matching_topic_count
})
self.render_jinja2_template("searchresults.html", template_values)
def get_classtime_table_old(self, students_data, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days=1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(
user_data_student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
query_video_logs = VideoLog.get_for_user_data_between_dts(
user_data_student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
rows = 0
chunks = 0
for i, user_data_student in enumerate(students_data):
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs,
key=lambda log: log.time_started())
rows += len(problem_and_video_logs)
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(
activity.time_started()) > (chunk_current.end +
self.chunk_delta):
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.user_data_student = user_data_student
chunk_current.start = self.dt_to_ctz(
activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()),
dt_end_ctz)
if chunk_current is not None:
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
column += 1
logging.info("old rows=" + str(rows) + ", old chunks=" + str(chunks))
classtime_table.balance()
return classtime_table
