def create_random_video_file(self):
"""
Helper function for testing video files.
"""
video_id = get_node_cache("Video").keys()[0]
youtube_id = get_node_cache("Video")[video_id][0]["youtube_id"]
fake_video_file = os.path.join(settings.CONTENT_ROOT, "%s.mp4" % youtube_id)
with open(fake_video_file, "w") as fh:
fh.write("")
self.assertTrue(os.path.exists(fake_video_file), "Make sure the video file was created, youtube_id='%s'." % youtube_id)
return (fake_video_file, video_id, youtube_id)
def exercise_dashboard(request):
slug = request.GET.get("topic")
if not slug:
title = _("Your Knowledge Map")
elif slug in topic_tools.get_node_cache("Topic"):
title = _(topic_tools.get_node_cache("Topic")[slug][0]["title"])
else:
raise Http404
context = {
"title": title,
}
return context
def get_playlist_entry_ids(cls, playlist):
"""Return a tuple of the playlist's video ids and exercise ids as sets"""
playlist_entries = playlist.get("children")
topic_cache = get_node_cache()["Topic"]
pl_video_ids = set([id for id in playlist_entries if topic_cache.get(id).get("kind") == "Video"])
pl_exercise_ids = set([id for id in playlist_entries if topic_cache.get(id).get("kind") == "Exercise"])
return (pl_video_ids, pl_exercise_ids)
def handle(self, *args, **options):
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
lang_code = lcode_to_ietf(options["lang_code"])
if lang_code not in AVAILABLE_EXERCISE_LANGUAGE_CODES:
logging.info("No exercises available for language %s" % lang_code)
else:
# Get list of exercises
exercise_ids = options["exercise_ids"].split(
",") if options["exercise_ids"] else None
exercise_ids = exercise_ids or ([
ex["id"]
for ex in get_topic_exercises(topic_id=options["topic_id"])
] if options["topic_id"] else None)
exercise_ids = exercise_ids or get_node_cache("Exercise").keys()
# Download the exercises
for exercise_id in exercise_ids:
scrape_exercise(exercise_id=exercise_id,
lang_code=lang_code,
force=options["force"])
logging.info("Process complete.")
def update_all_distributed_callback(request):
"""
"""
if request.method != "POST":
raise PermissionDenied("Only POST allowed to this URL endpoint.")
videos = json.loads(request.POST["video_logs"])
exercises = json.loads(request.POST["exercise_logs"])
user = FacilityUser.objects.get(id=request.POST["user_id"])
node_cache = get_node_cache()
# Save videos
n_videos_uploaded = 0
for video in videos:
video_id = video['video_id']
youtube_id = video['youtube_id']
# Only save video logs for videos that we recognize.
if video_id not in node_cache["Content"]:
logging.warn("Skipping unknown video %s" % video_id)
continue
try:
(vl, _) = VideoLog.get_or_initialize(user=user, video_id=video_id) # has to be that video_id, could be any youtube_id
for key,val in video.iteritems():
setattr(vl, key, val)
logging.debug("Saving video log for %s: %s" % (video_id, vl))
vl.save()
n_videos_uploaded += 1
except KeyError: #
logging.error("Could not save video log for data with missing values: %s" % video)
except Exception as e:
error_message = _("Unexpected error importing videos: %(err_msg)s") % {"err_msg": e}
return JsonResponseMessageError(error_message)
# Save exercises
n_exercises_uploaded = 0
for exercise in exercises:
# Only save video logs for videos that we recognize.
if exercise['exercise_id'] not in node_cache['Exercise']:
logging.warn("Skipping unknown video %s" % exercise['exercise_id'])
continue
try:
(el, _) = ExerciseLog.get_or_initialize(user=user, exercise_id=exercise["exercise_id"])
for key,val in exercise.iteritems():
setattr(el, key, val)
logging.debug("Saving exercise log for %s: %s" % (exercise['exercise_id'], el))
el.save()
n_exercises_uploaded += 1
except KeyError:
logging.error("Could not save exercise log for data with missing values: %s" % exercise)
except Exception as e:
error_message = _("Unexpected error importing exercises: %(err_msg)s") % {"err_msg": e}
return JsonResponseMessageError(error_message)
return JsonResponseMessageSuccess(_("Uploaded %(num_exercises)d exercises and %(num_videos)d videos") % {
"num_exercises": n_exercises_uploaded,
"num_videos": n_videos_uploaded,
})
def update_all_distributed_callback(request):
"""
"""
if request.method != "POST":
raise PermissionDenied("Only POST allowed to this URL endpoint.")
videos = json.loads(request.POST["video_logs"])
exercises = json.loads(request.POST["exercise_logs"])
user = FacilityUser.objects.get(id=request.POST["user_id"])
node_cache = get_node_cache()
# Save videos
n_videos_uploaded = 0
for video in videos:
video_id = video['video_id']
youtube_id = video['youtube_id']
# Only save video logs for videos that we recognize.
if video_id not in node_cache["Video"]:
logging.warn("Skipping unknown video %s" % video_id)
continue
try:
(vl, _) = VideoLog.get_or_initialize(user=user, video_id=video_id) # has to be that video_id, could be any youtube_id
for key,val in video.iteritems():
setattr(vl, key, val)
logging.debug("Saving video log for %s: %s" % (video_id, vl))
vl.save()
n_videos_uploaded += 1
except KeyError: #
logging.error("Could not save video log for data with missing values: %s" % video)
except Exception as e:
error_message = _("Unexpected error importing videos: %(err_msg)s") % {"err_msg": e}
return JsonResponseMessageError(error_message)
# Save exercises
n_exercises_uploaded = 0
for exercise in exercises:
# Only save video logs for videos that we recognize.
if exercise['exercise_id'] not in node_cache['Exercise']:
logging.warn("Skipping unknown video %s" % exercise['exercise_id'])
continue
try:
(el, _) = ExerciseLog.get_or_initialize(user=user, exercise_id=exercise["exercise_id"])
for key,val in exercise.iteritems():
setattr(el, key, val)
logging.debug("Saving exercise log for %s: %s" % (exercise['exercise_id'], el))
el.save()
n_exercises_uploaded += 1
except KeyError:
logging.error("Could not save exercise log for data with missing values: %s" % exercise)
except Exception as e:
error_message = _("Unexpected error importing exercises: %(err_msg)s") % {"err_msg": e}
return JsonResponseMessageError(error_message)
return JsonResponseMessageSuccess(_("Uploaded %(num_exercises)d exercises and %(num_videos)d videos") % {
"num_exercises": n_exercises_uploaded,
"num_videos": n_videos_uploaded,
})
def set_unit_navigate_to_exercise(self, unit, exercise_id):
"""
Set the student unit. Navigate to an exercise.
"""
set_current_unit_settings_value(self.facility.id, unit)
self.browse_to(self.live_server_url + get_node_cache("Exercise")[exercise_id]["path"])
def clean_exercise_id(self):
"""
Make sure the exercise ID is found.
"""
if not self.cleaned_data.get("exercise_id",
"") in get_node_cache('Exercise'):
raise forms.ValidationError(_("Exercise ID not recognized"))
def set_unit_navigate_to_exercise(self, unit, exercise_id):
"""
Set the student unit. Navigate to an exercise.
"""
set_current_unit_settings_value(self.facility.id, unit)
self.browse_to(self.live_server_url +
get_node_cache("Exercise")[exercise_id]["path"])
def show_cache(self, force=False):
"""Go through each cacheable page, and show which are cached and which are NOT"""
for node_type in ['Topic', 'Video', 'Exercise']:
self.stdout.write("Cached %ss:\n" % node_type)
for narr in topic_tools.get_node_cache(node_type).values():
for n in narr:
if caching.has_cache_key(path=n["path"]):
self.stdout.write("\t%s\n" % n["path"])
def show_cache(self, force=False):
"""Go through each cacheable page, and show which are cached and which are NOT"""
for node_type in ['Topic', 'Content', 'Exercise']:
self.stdout.write("Cached %ss:\n" % node_type)
for narr in topic_tools.get_node_cache(node_type).values():
for n in narr:
if caching.has_cache_key(path=n["path"]):
self.stdout.write("\t%s\n" % n["path"])
def test_topic_availability(self):
for node_list in get_node_cache("Topic").values():
for topic in node_list:
if "Exercise" in topic["contains"]:
self.assertTrue(topic["available"], "Make sure all topics containing exercises are shown as available.")
if topic["children"] and len(topic["contains"]) == 1 and "Video" in topic["contains"]:
any_on_disk = bool(sum([v["on_disk"] for v in topic["children"]]))
self.assertEqual(topic["available"], any_on_disk, "Make sure topic availability matches video availability when only videos are available.")
def knowledge_map_json(request, topic_id):
"""
Topic nodes can now have a "knowledge_map" stamped on them.
This code currently exposes that data to the kmap-editor code,
mostly as it expects it now.
So this is kind of a hack-ish mix of code that avoids rewriting kmap-editor.js,
but allows a cleaner rewrite of the stored data, and bridges the gap between
that messiness and the cleaner back-end.
"""
# Try and get the requested topic, and make sure it has knowledge map data available.
topic = get_node_cache("Topic").get(topic_id)
if not topic:
raise Http404("Topic '%s' not found" % topic_id)
elif not "knowledge_map" in topic[0]:
raise Http404("Topic '%s' has no knowledge map metadata." % topic_id)
# For each node (can be of any type now), pull out only
# the relevant data.
kmap = topic[0]["knowledge_map"]
nodes_out = {}
for id, kmap_data in kmap["nodes"].iteritems():
cur_node = get_node_cache(kmap_data["kind"])[id][0]
nodes_out[id] = {
"id": cur_node["id"],
"title": _(cur_node["title"]),
"h_position": kmap_data["h_position"],
"v_position": kmap_data["v_position"],
"icon_url": cur_node.get("icon_url",
cur_node.get("icon_src")), # messy
"path": cur_node["path"],
}
if not "polylines" in kmap: # messy
# Two ways to define lines:
# 1. have "polylines" defined explicitly
# 2. use prerequisites to compute lines on the fly.
nodes_out[id]["prerequisites"] = cur_node.get("prerequisites", [])
return JsonResponse({
"nodes": nodes_out,
"polylines": kmap.get("polylines"), # messy
})
def clear_cache(self):
"""Go through each cacheable page, and show which are cached and which are NOT"""
for node_type in ['Topic', 'Content', 'Exercise']:
self.stdout.write("Clearing %ss:\n" % node_type)
for narr in topic_tools.get_node_cache(node_type).values():
for n in narr:
if caching.has_cache_key(path=n["path"]):
self.stdout.write("\t%s\n" % n["path"])
caching.expire_page(path=n["path"])
def setUp(self):
"""
Create a student, log the student in, and go to the exercise page.
"""
super(StudentExerciseTest, self).setUp()
self.student = self.create_student(facility_name=self.facility_name)
self.browser_login_student(self.student_username, self.student_password, facility_name=self.facility_name)
self.browse_to(self.live_server_url + get_node_cache("Exercise")[self.EXERCISE_SLUG][0]["path"])
self.browser_check_django_message(num_messages=0) # make sure no messages
def clear_cache(self):
"""Go through each cacheable page, and show which are cached and which are NOT"""
for node_type in ['Topic', 'Video', 'Exercise']:
self.stdout.write("Clearing %ss:\n" % node_type)
for narr in topic_tools.get_node_cache(node_type).values():
for n in narr:
if caching.has_cache_key(path=n["path"]):
self.stdout.write("\t%s\n" % n["path"])
caching.expire_page(path=n["path"])
def knowledge_map_json(request, topic_id):
"""
Topic nodes can now have a "knowledge_map" stamped on them.
This code currently exposes that data to the kmap-editor code,
mostly as it expects it now.
So this is kind of a hack-ish mix of code that avoids rewriting kmap-editor.js,
but allows a cleaner rewrite of the stored data, and bridges the gap between
that messiness and the cleaner back-end.
"""
# Try and get the requested topic, and make sure it has knowledge map data available.
topic = get_node_cache("Topic").get(topic_id)
if not topic:
raise Http404("Topic '%s' not found" % topic_id)
elif not "knowledge_map" in topic[0]:
raise Http404("Topic '%s' has no knowledge map metadata." % topic_id)
# For each node (can be of any type now), pull out only
# the relevant data.
kmap = topic[0]["knowledge_map"]
nodes_out = {}
for id, kmap_data in kmap["nodes"].iteritems():
cur_node = get_node_cache(kmap_data["kind"])[id][0]
nodes_out[id] = {
"id": cur_node["id"],
"title": _(cur_node["title"]),
"h_position": kmap_data["h_position"],
"v_position": kmap_data["v_position"],
"icon_url": cur_node.get("icon_url", cur_node.get("icon_src")), # messy
"path": cur_node["path"],
}
if not "polylines" in kmap: # messy
# Two ways to define lines:
# 1. have "polylines" defined explicitly
# 2. use prerequisites to compute lines on the fly.
nodes_out[id]["prerequisites"] = cur_node.get("prerequisites", [])
return JsonResponse({
"nodes": nodes_out,
"polylines": kmap.get("polylines"), # messy
})
def save_exercise_log(request):
"""
Receives an exercise_id and relevant data,
saves it to the currently authorized user.
"""
# Form does all data validation, including of the exercise_id
form = ExerciseLogForm(data=simplejson.loads(request.body))
if not form.is_valid():
raise Exception(form.errors)
data = form.data
# More robust extraction of previous object
user = request.session["facility_user"]
(exerciselog, was_created) = ExerciseLog.get_or_initialize(
user=user, exercise_id=data["exercise_id"])
previously_complete = exerciselog.complete
exerciselog.attempts = data[
"attempts"] # don't increment, because we fail to save some requests
exerciselog.streak_progress = data["streak_progress"]
exerciselog.points = data["points"]
exerciselog.language = data.get("language") or request.language
try:
exerciselog.full_clean()
exerciselog.save(update_userlog=True)
except ValidationError as e:
return JsonResponseMessageError(
_("Could not save ExerciseLog") + u": %s" % e)
if "points" in request.session:
del request.session["points"] # will be recomputed when needed
# Special message if you've just completed.
# NOTE: it's important to check this AFTER calling save() above.
if not previously_complete and exerciselog.complete:
exercise = get_node_cache("Exercise").get(data["exercise_id"],
[None])[0]
junk, next_exercise = get_neighbor_nodes(
exercise, neighbor_kind="Exercise") if exercise else None
if not next_exercise:
return JsonResponseMessageSuccess(
_("You have mastered this exercise and this topic!"))
else:
return JsonResponseMessageSuccess(
_("You have mastered this exercise! Please continue on to <a href='%(href)s'>%(title)s</a>"
) % {
"href": next_exercise["path"],
"title": _(next_exercise["title"]),
})
# Return no message in release mode; "data saved" message in debug mode.
return JsonResponse({})
def search(request):
# Inputs
query = request.GET.get('query')
category = request.GET.get('category')
max_results_per_category = request.GET.get('max_results', 25)
# Outputs
query_error = None
possible_matches = {}
hit_max = {}
if query is None:
query_error = _("Error: query not specified.")
# elif len(query) < 3:
# query_error = _("Error: query too short.")
else:
query = query.lower()
# search for topic, video or exercise with matching title
nodes = []
for node_type, node_dict in topic_tools.get_node_cache().iteritems():
if category and node_type != category:
# Skip categories that don't match (if specified)
continue
possible_matches[node_type] = [] # make dict only for non-skipped categories
for node in node_dict.values():
title = _(node['title']).lower() # this could be done once and stored.
keywords = [x.lower() for x in node.get('keywords', [])]
tags = [x.lower() for x in node.get('tags', [])]
if title == query:
# Redirect to an exact match
return HttpResponseRedirect(reverse('learn') + node['path'])
elif (len(possible_matches[node_type]) < max_results_per_category and
(query in title or query in keywords or query in tags)):
# For efficiency, don't do substring matches when we've got lots of results
possible_matches[node_type].append(node)
hit_max[node_type] = len(possible_matches[node_type]) == max_results_per_category
return {
'title': _("Search results for '%(query)s'") % {"query": (query if query else "")},
'query_error': query_error,
'results': possible_matches,
'hit_max': hit_max,
'query': query,
'max_results': max_results_per_category,
'category': category,
}
def get_playlist_entries(playlist, entry_type, language=settings.LANGUAGE_CODE):
"""
Given a VanillaPlaylist, inspect its 'entries' attribute and return a list
containing corresponding nodes for each item from the topic tree.
entry_type should be "Exercise" or "Video".
"""
unprepared = filter(lambda e: e["entity_kind"] == entry_type, playlist.entries)
prepared = []
for entry in unprepared:
new_item = get_node_cache(language=language)[entry_type].get(entry["entity_id"], None)
if new_item:
prepared.append(new_item)
return prepared
def get_playlist_entries(playlist, entry_type, language=settings.LANGUAGE_CODE):
"""
Given a VanillaPlaylist, inspect its 'entries' attribute and return a list
containing corresponding nodes for each item from the topic tree.
entry_type should be "Exercise" or "Video".
"""
unprepared = filter(lambda e: e["entity_kind"]==entry_type, playlist.entries)
prepared = []
for entry in unprepared:
new_item = get_node_cache(language=language)[entry_type].get(entry['entity_id'], None)
if new_item:
prepared.append(new_item)
return prepared
def get_playlist_entry_ids(cls, playlist):
"""Return a tuple of the playlist's video ids and exercise ids as sets"""
playlist_entries = playlist.get("children")
topic_cache = get_node_cache()["Topic"]
pl_video_ids = set([
id for id in playlist_entries
if topic_cache.get(id).get("kind") == "Video"
])
pl_exercise_ids = set([
id for id in playlist_entries
if topic_cache.get(id).get("kind") == "Exercise"
])
return (pl_video_ids, pl_exercise_ids)
def setUp(self):
"""
Create a student, log the student in, and go to the exercise page.
"""
super(StudentExerciseTest, self).setUp()
self.facility_name = "fac"
self.facility = self.create_facility(name=self.facility_name)
self.student = self.create_student(username=self.student_username,
password=self.student_password,
facility=self.facility)
self.browser_login_student(self.student_username, self.student_password, facility_name=self.facility_name)
self.browse_to(self.live_server_url + reverse("learn") + get_node_cache("Exercise")[self.EXERCISE_SLUG]["path"])
self.nanswers = self.browser.execute_script('return window.ExerciseParams.STREAK_CORRECT_NEEDED;')
def setUp(self):
"""
Create a student, log the student in, and go to the exercise page.
"""
super(StudentExerciseTest, self).setUp()
self.facility_name = "fac"
self.facility = self.create_facility(name=self.facility_name)
self.student = self.create_student(
username=self.student_username, password=self.student_password, facility=self.facility
)
self.browser_login_student(self.student_username, self.student_password, facility_name=self.facility_name)
self.browse_to(self.live_server_url + reverse("learn") + get_node_cache("Exercise")[self.EXERCISE_SLUG]["path"])
self.nanswers = self.browser.execute_script("return window.ExerciseParams.STREAK_CORRECT_NEEDED;")
def test_topic_availability(self):
for topic in get_node_cache("Topic").values():
if "Exercise" in topic["contains"]:
self.assertTrue(
topic["available"],
"Make sure all topics containing exercises are shown as available."
)
if topic["children"] and "Video" in topic["contains"]:
any_available = bool(
sum([v.get("available", False)
for v in topic["children"]]))
self.assertEqual(
topic["available"], any_available,
"Make sure topic availability matches video availability when only videos are available."
)
def setUp(self):
"""
Create a student, log the student in, and go to the exercise page.
"""
super(StudentExerciseTest, self).setUp()
self.student = self.create_student(facility_name=self.facility_name)
self.browser_login_student(self.student_username,
self.student_password,
facility_name=self.facility_name)
self.browse_to(
self.live_server_url +
get_node_cache("Exercise")[self.EXERCISE_SLUG][0]["path"])
self.browser_check_django_message(
num_messages=0) # make sure no messages
def save_exercise_log(request):
"""
Receives an exercise_id and relevant data,
saves it to the currently authorized user.
"""
# Form does all data validation, including of the exercise_id
form = ExerciseLogForm(data=simplejson.loads(request.body))
if not form.is_valid():
raise Exception(form.errors)
data = form.data
# More robust extraction of previous object
user = request.session["facility_user"]
(exerciselog, was_created) = ExerciseLog.get_or_initialize(user=user, exercise_id=data["exercise_id"])
previously_complete = exerciselog.complete
exerciselog.attempts = data["attempts"] # don't increment, because we fail to save some requests
exerciselog.streak_progress = data["streak_progress"]
exerciselog.points = data["points"]
exerciselog.language = data.get("language") or request.language
try:
exerciselog.full_clean()
exerciselog.save(update_userlog=True)
except ValidationError as e:
return JsonResponseMessageError(_("Could not save ExerciseLog") + u": %s" % e)
if "points" in request.session:
del request.session["points"] # will be recomputed when needed
# Special message if you've just completed.
# NOTE: it's important to check this AFTER calling save() above.
if not previously_complete and exerciselog.complete:
exercise = get_node_cache("Exercise").get(data["exercise_id"], [None])[0]
junk, next_exercise = get_neighbor_nodes(exercise, neighbor_kind="Exercise") if exercise else None
if not next_exercise:
return JsonResponseMessageSuccess(_("You have mastered this exercise and this topic!"))
else:
return JsonResponseMessageSuccess(_("You have mastered this exercise! Please continue on to <a href='%(href)s'>%(title)s</a>") % {
"href": next_exercise["path"],
"title": _(next_exercise["title"]),
})
# Return no message in release mode; "data saved" message in debug mode.
return JsonResponse({})
def handle(self, *args, **options):
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
lang_code = lcode_to_ietf(options["lang_code"])
if lang_code not in AVAILABLE_EXERCISE_LANGUAGE_CODES:
logging.info("No exercises available for language %s" % lang_code)
else:
# Get list of exercises
exercise_ids = options["exercise_ids"].split(",") if options["exercise_ids"] else None
exercise_ids = exercise_ids or ([ex["id"] for ex in get_topic_exercises(topic_id=options["topic_id"])] if options["topic_id"] else None)
exercise_ids = exercise_ids or get_node_cache("Exercise").keys()
# Download the exercises
for exercise_id in exercise_ids:
scrape_exercise(exercise_id=exercise_id, lang_code=lang_code, force=options["force"])
logging.info("Process complete.")
def _setup(self, num_logs=50, **kwargs):
super(OneHundredRandomLogUpdates, self)._setup(**kwargs)
node_cache = get_node_cache()
try:
self.user = FacilityUser.objects.get(username=self.username)
except:
#take username from ExerciseLog
all_exercises = ExerciseLog.objects.all()
self.user = FacilityUser.objects.get(id=all_exercises[0].user_id)
print self.username, " not in FacilityUsers, using ", self.user
self.num_logs = num_logs
#give the platform a chance to cache the logs
ExerciseLog.objects.filter(user=self.user).delete()
for x in range(num_logs):
while True:
ex_idx = int(self.random.random() *
len(node_cache["Exercise"].keys()))
ex_id = node_cache["Exercise"].keys()[ex_idx]
if not ExerciseLog.objects.filter(user=self.user,
exercise_id=ex_id):
break
ex = ExerciseLog(user=self.user, exercise_id=ex_id)
ex.save()
self.exercise_list = ExerciseLog.objects.filter(user=self.user)
self.exercise_count = self.exercise_list.count()
VideoLog.objects.filter(user=self.user).delete()
for x in range(num_logs):
while True:
vid_idx = int(self.random.random() *
len(node_cache["Content"].keys()))
vid_id = node_cache["Content"].keys()[vid_idx]
if not VideoLog.objects.filter(user=self.user,
video_id=vid_id):
break
vid = VideoLog(user=self.user, video_id=vid_id)
vid.save()
self.video_list = VideoLog.objects.filter(user=self.user)
self.video_count = self.video_list.count()
def handle(self, *args, **kwargs):
MALFORMED_IDS = []
video_slugs = set(video_dict_by_video_id().keys())
exercise_slugs = set(get_node_cache()["Exercise"].keys())
all_playlists = json.load(open(os.path.join(settings.PROJECT_PATH, 'playlist/playlists.json')))
# for pl in Playlist.all():
for pl in all_playlists:
# entries = pl.entries
entries = pl.get("entries")
# Find video ids in the playlists that are not in the topic tree
video_entry_slugs = [enforce_and_strip_slug(pl.get("id"), e['entity_id']) for e in entries if e['entity_kind'] == 'Video']
nonexistent_video_slugs = set(filter(None, video_entry_slugs)) - video_slugs
# Find exercise ids in the playlists that are not in the topic tree
ex_entry_slugs = [enforce_and_strip_slug(pl.get("id"), e['entity_id']) for e in entries if e['entity_kind'] == 'Exercise']
nonexistent_ex_slugs = set(filter(None, ex_entry_slugs)) - exercise_slugs
# Print malformed videos
for slug in nonexistent_video_slugs:
errormsg = "Video slug in playlist {0} not found in videos: {1}"
# print errormsg.format(pl.id, slug)
print errormsg.format(pl.get("id"), slug)
# Print malformed exercises
for slug in nonexistent_ex_slugs:
errormsg = "Exercise slug in playlist {0} not found in exercises: {1}"
# print errormsg.format(pl.id, slug)
print errormsg.format(pl.get("id"), slug)
# Print misspelled ids
for m in MALFORMED_IDS:
errormsg = "Malformed slug in playlist {0}. Please investigate: {1}"
# print errormsg.format(pl.id, slug)
print errormsg.format(pl.get("id"), m)
MALFORMED_IDS = []
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the distributed server.")
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
#
ensure_dir(settings.CONTENT_ROOT)
# Get list of videos
lang_code = lcode_to_ietf(options["lang_code"])
video_map = get_dubbed_video_map(lang_code) or {}
video_ids = options["video_ids"].split(",") if options["video_ids"] else None
video_ids = video_ids or (
[vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None
)
video_ids = video_ids or video_map.keys()
# Download the videos
for video_id in video_ids:
if video_id in video_map:
youtube_id = video_map[video_id]
elif video_id in video_map.values():
# Perhaps they sent in a youtube ID? We can handle that!
youtube_id = video_id
else:
logging.error("No mapping for video_id=%s; skipping" % video_id)
continue
try:
scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"])
# scrape_thumbnail(youtube_id=youtube_id)
logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"]))
except Exception as e:
logging.error("Failed to download video %s: %s" % (youtube_id, e))
logging.info("Process complete.")
def add_full_title_from_topic_tree(entry, video_title_dict):
# TODO (aron): Add i18n by varying the language of the topic tree here
topictree = get_node_cache()
entry_kind = entry['entity_kind']
entry_name = entry['entity_id']
try:
if entry_kind == 'Exercise':
nodedict = topictree['Exercise']
elif entry_kind == 'Video':
# TODO-blocker: move use of video_dict_by_id to slug2id_map
nodedict = video_title_dict
else:
nodedict = {}
entry['title'] = nodedict[entry_name]['title']
entry['description'] = nodedict[entry_name].get('description', '')
except KeyError:
# TODO: edit once we get the properly labeled entity ids from Nalanda
entry['title'] = entry['description'] = entry['entity_id']
return entry
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the distributed server.")
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
#
ensure_dir(settings.CONTENT_ROOT)
# Get list of videos
lang_code = lcode_to_ietf(options["lang_code"])
video_map = get_dubbed_video_map(lang_code) or {}
video_ids = options["video_ids"].split(",") if options["video_ids"] else None
video_ids = video_ids or ([vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None)
video_ids = video_ids or video_map.keys()
# Download the videos
for video_id in video_ids:
if video_id in video_map:
youtube_id = video_map[video_id]
elif video_id in video_map.values():
# Perhaps they sent in a youtube ID? We can handle that!
youtube_id = video_id
else:
logging.error("No mapping for video_id=%s; skipping" % video_id)
continue
try:
scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"])
#scrape_thumbnail(youtube_id=youtube_id)
logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"]))
except Exception as e:
logging.error("Failed to download video %s: %s" % (youtube_id, e))
logging.info("Process complete.")
def _setup(self, num_logs=50, **kwargs):
super(OneHundredRandomLogUpdates, self)._setup(**kwargs)
node_cache = get_node_cache()
try:
self.user = FacilityUser.objects.get(username=self.username)
except:
#take username from ExerciseLog
all_exercises = ExerciseLog.objects.all()
self.user = FacilityUser.objects.get(id=all_exercises[0].user_id)
print self.username, " not in FacilityUsers, using ", self.user
self.num_logs = num_logs
#give the platform a chance to cache the logs
ExerciseLog.objects.filter(user=self.user).delete()
for x in range(num_logs):
while True:
ex_idx = int(self.random.random() * len(node_cache["Exercise"].keys()))
ex_id = node_cache["Exercise"].keys()[ex_idx]
if not ExerciseLog.objects.filter(user=self.user, exercise_id=ex_id):
break
ex = ExerciseLog(user=self.user, exercise_id=ex_id)
ex.save()
self.exercise_list = ExerciseLog.objects.filter(user=self.user)
self.exercise_count = self.exercise_list.count()
VideoLog.objects.filter(user=self.user).delete()
for x in range(num_logs):
while True:
vid_idx = int(self.random.random() * len(node_cache["Content"].keys()))
vid_id = node_cache["Content"].keys()[vid_idx]
if not VideoLog.objects.filter(user=self.user, video_id=vid_id):
break
vid = VideoLog(user=self.user, video_id=vid_id)
vid.save()
self.video_list = VideoLog.objects.filter(user=self.user)
self.video_count = self.video_list.count()
def create_cache(self, force=False):
for node_type in ['Topic', 'Video', 'Exercise']:
self.stdout.write("Caching %ss:\n" % node_type)
for narr in topic_tools.get_node_cache(node_type).values():
for n in narr:
self.create_page_cache(path=n["path"], force=force)
def clean_video_id(self):
"""
Make sure the video ID is found.
"""
if self.cleaned_data["video_id"] not in get_node_cache("Video"):
raise forms.ValidationError(_("Video ID not recognized."))
def update_all_central_callback(request):
"""
Callback after authentication.
Parses out the request token verification.
Then finishes the request by getting an auth token.
"""
if not "ACCESS_TOKEN" in request.session:
finish_auth(request)
exercises = get_api_resource(request, "/api/v1/user/exercises")
videos = get_api_resource(request, "/api/v1/user/videos")
node_cache = get_node_cache()
# Collate videos
video_logs = []
for video in videos:
# Assume that KA videos are all english-language, not dubbed (for now)
youtube_id = video.get('video', {}).get('youtube_id', "")
video_id = get_video_id(youtube_id) # map from youtube_id to video_id (across all languages)
# Only save videos with progress
if not video.get('seconds_watched', None):
continue
# Only save video logs for videos that we recognize.
if video_id not in node_cache["Video"]:
logging.warn("Skipping unknown video %s" % video_id)
continue
try:
video_logs.append({
"video_id": video_id,
"youtube_id": youtube_id,
"total_seconds_watched": video['seconds_watched'],
"points": VideoLog.calc_points(video['seconds_watched'], video['duration']),
"complete": video['completed'],
"completion_timestamp": convert_ka_date(video['last_watched']) if video['completed'] else None,
})
logging.debug("Got video log for %s: %s" % (video_id, video_logs[-1]))
except KeyError: #
logging.error("Could not save video log for data with missing values: %s" % video)
# Collate exercises
exercise_logs = []
for exercise in exercises:
# Only save exercises that have any progress.
if not exercise.get('last_done', None):
continue
# Only save video logs for videos that we recognize.
slug = exercise.get('exercise', "")
if slug not in node_cache['Exercise']:
logging.warn("Skipping unknown video %s" % slug)
continue
try:
completed = exercise['streak'] >= 10
basepoints = node_cache['Exercise'][slug][0]['basepoints']
exercise_logs.append({
"exercise_id": slug,
"streak_progress": min(100, 100 * exercise['streak']/10), # duplicates logic elsewhere
"attempts": exercise['total_done'],
"points": ExerciseLog.calc_points(basepoints, ncorrect=exercise['streak'], add_randomness=False), # no randomness when importing from KA
"complete": completed,
"attempts_before_completion": exercise['total_done'] if not exercise['practiced'] else None, #can't figure this out if they practiced after mastery.
"completion_timestamp": convert_ka_date(exercise['proficient_date']) if completed else None,
})
logging.debug("Got exercise log for %s: %s" % (slug, exercise_logs[-1]))
except KeyError:
logging.error("Could not save exercise log for data with missing values: %s" % exercise)
# POST the data back to the distributed server
try:
dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None
logging.debug("POST'ing to %s" % request.session["distributed_callback_url"])
response = requests.post(
request.session["distributed_callback_url"],
cookies={ "csrftoken": request.session["distributed_csrf_token"] },
data = {
"csrfmiddlewaretoken": request.session["distributed_csrf_token"],
"video_logs": json.dumps(video_logs, default=dthandler),
"exercise_logs": json.dumps(exercise_logs, default=dthandler),
"user_id": request.session["distributed_user_id"],
}
)
logging.debug("Response (%d): %s" % (response.status_code, response.content))
except requests.exceptions.ConnectionError as e:
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": "error",
"message": _("Could not connect to your KA Lite installation to share Khan Academy data."),
"message_id": "id_khanload",
}))
except Exception as e:
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": "error",
"message": _("Failure to send data to your KA Lite installation: %s") % e,
"message_id": "id_khanload",
}))
try:
json_response = json.loads(response.content)
if not isinstance(json_response, dict) or len(json_response) != 1:
# Could not validate the message is a single key-value pair
raise Exception(_("Unexpected response format from your KA Lite installation."))
message_type = json_response.keys()[0]
message = json_response.values()[0]
except ValueError as e:
message_type = "error"
message = unicode(e)
except Exception as e:
message_type = "error"
message = _("Loading json object: %s") % e
# If something broke on the distributed server, we have no way to recover.
# For now, just show the error to users.
#
# Ultimately, we have a message, would like to share with the distributed server.
# if response.status_code != 200:
# return HttpResponseServerError(response.content)
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": message_type,
"message": message,
"message_id": "id_khanload",
}))
def compute_data(data_types, who, where, language=settings.LANGUAGE_CODE):
"""
Compute the data in "data_types" for each user in "who", for the topics selected by "where"
who: list of users
where: topic_path
data_types can include:
pct_mastery
effort
attempts
"""
# None indicates that the data hasn't been queried yet.
# We'll query it on demand, for efficiency
topics = None
exercises = None
videos = None
# Initialize an empty dictionary of data, video logs, exercise logs, for each user
data = OrderedDict(
zip([w.id for w in who],
[dict()
for i in range(len(who))])) # maintain the order of the users
vid_logs = dict(zip([w.id for w in who], [[] for i in range(len(who))]))
ex_logs = dict(zip([w.id for w in who], [[] for i in range(len(who))]))
if UserLog.is_enabled():
activity_logs = dict(
zip([w.id for w in who], [[] for i in range(len(who))]))
# Set up queries (but don't run them), so we have really easy aliases.
# Only do them if they haven't been done yet (tell this by passing in a value to the lambda function)
# Topics: topics.
# Exercises: names (ids for ExerciseLog objects)
# Videos: video_id (ids for VideoLog objects)
# This lambda partial creates a function to return all items with a particular path from the NODE_CACHE.
search_fun_single_path = partial(lambda t, p: t["path"].startswith(p),
p=tuple(where))
# This lambda partial creates a function to return all items with paths matching a list of paths from NODE_CACHE.
search_fun_multi_path = partial(
lambda ts, p: any([t["path"].startswith(p) for t in ts]),
p=tuple(where))
# Functions that use the functions defined above to return topics, exercises, and videos based on paths.
query_topics = partial(lambda t, sf: t if t is not None else [
t["id"]
for t in filter(sf,
get_node_cache('Topic', language=language).values())
],
sf=search_fun_single_path)
query_exercises = partial(lambda e, sf: e if e is not None else [
ex["id"]
for ex in filter(sf,
get_exercise_cache(language=language).values())
],
sf=search_fun_single_path)
query_videos = partial(lambda v, sf: v if v is not None else [
vid["id"] for vid in filter(
sf,
get_node_cache('Content', language=language).values())
],
sf=search_fun_single_path)
# No users, don't bother.
if len(who) > 0:
# Query out all exercises, videos, exercise logs, and video logs before looping to limit requests.
# This means we could pull data for n-dimensional coach report displays with the same number of requests!
# Note: User activity is polled inside the loop, to prevent possible slowdown for exercise and video reports.
exercises = query_exercises(exercises)
videos = query_videos(videos)
if exercises:
ex_logs = query_logs(data.keys(), exercises, "exercise", ex_logs)
if videos:
vid_logs = query_logs(data.keys(), videos, "video", vid_logs)
for data_type in (data_types if not hasattr(data_types, "lower") else [
data_types
]): # convert list from string, if necessary
if data_type in data[data.keys(
)[0]]: # if the first user has it, then all do; no need to calc again.
continue
#
# These are summary stats: you only get one per user
#
if data_type == "pct_mastery":
# Efficient query out, spread out to dict
for user in data.keys():
data[user][
data_type] = 0 if not ex_logs[user] else 100. * sum(
[el['complete']
for el in ex_logs[user]]) / float(len(exercises))
elif data_type == "effort":
if "ex:attempts" in data[data.keys(
)[0]] and "vid:total_seconds_watched" in data[data.keys()[0]]:
# exercises and videos would be initialized already
for user in data.keys():
avg_attempts = 0 if len(exercises) == 0 else sum(
data[user]["ex:attempts"].values()) / float(
len(exercises))
avg_video_points = 0 if len(videos) == 0 else sum(
data[user]["vid:total_seconds_watched"].values(
)) / float(len(videos))
data[user][data_type] = 100. * (
0.5 * avg_attempts / 10. +
0.5 * avg_video_points / 750.)
else:
data_types += [
"ex:attempts", "vid:total_seconds_watched", "effort"
]
#
# These are detail stats: you get many per user
#
# Just querying out data directly: Video
elif data_type.startswith("vid:") and data_type[4:] in [
f.name for f in VideoLog._meta.fields
]:
for user in data.keys():
data[user][data_type] = OrderedDict([
(v['video_id'], v[data_type[4:]])
for v in vid_logs[user]
])
# Just querying out data directly: Exercise
elif data_type.startswith("ex:") and data_type[3:] in [
f.name for f in ExerciseLog._meta.fields
]:
for user in data.keys():
data[user][data_type] = OrderedDict([
(el['exercise_id'], el[data_type[3:]])
for el in ex_logs[user]
])
# User Log Queries
elif data_type.startswith("user:"******"", "activity",
activity_logs)
for user in data.keys():
data[user][data_type] = [
log[data_type[5:]] for log in activity_logs[user]
]
# User Summary Queries
elif data_type.startswith("usersum:") and data_type[8:] in [
f.name for f in UserLogSummary._meta.fields
] and UserLog.is_enabled():
activity_logs = query_logs(data.keys(), "", "summaryactivity",
activity_logs)
for user in data.keys():
data[user][data_type] = sum(
[log[data_type[8:]] for log in activity_logs[user]])
# Unknown requested quantity
else:
raise Exception(
"Unknown type: '%s' not in %s" %
(data_type, str([f.name
for f in ExerciseLog._meta.fields])))
# Returning empty list instead of None allows javascript on client
# side to read 'length' property without error.
exercises = exercises or []
videos = videos or []
return {
"data": data,
"topics": topics,
"exercises": exercises,
"videos": videos,
}
def api_data(request, xaxis="", yaxis=""):
"""Request contains information about what data are requested (who, what, and how).
Response should be a JSON object
* data contains the data, structred by user and then datatype
* the rest of the data is metadata, useful for displaying detailed info about data.
"""
language = lcode_to_django_lang(request.language)
# Get the request form
try:
form = get_data_form(request, xaxis=xaxis,
yaxis=yaxis) # (data=request.REQUEST)
except Exception as e:
# In investigating #1509: we can catch SQL errors here and communicate clearer error
# messages with the user here. For now, we have no such error to catch, so just
# pass the errors on to the user (via the @api_handle_error_with_json decorator).
raise e
# Query out the data: who?
if form.data.get("user"):
facility = []
groups = []
users = [get_object_or_404(FacilityUser, id=form.data.get("user"))]
elif form.data.get("group"):
facility = []
if form.data.get("group") == "Ungrouped":
groups = []
users = FacilityUser.objects.filter(
facility__in=[form.data.get("facility")],
group__isnull=True,
is_teacher=False).order_by("last_name", "first_name")
else:
groups = [
get_object_or_404(FacilityGroup, id=form.data.get("group"))
]
users = FacilityUser.objects.filter(group=form.data.get("group"),
is_teacher=False).order_by(
"last_name", "first_name")
elif form.data.get("facility"):
facility = get_object_or_404(Facility, id=form.data.get("facility"))
groups = FacilityGroup.objects.filter(
facility__in=[form.data.get("facility")])
users = FacilityUser.objects.filter(
facility__in=[form.data.get("facility")],
is_teacher=False).order_by("last_name", "first_name")
else:
# Allow superuser to see the data.
if request.user.is_authenticated() and request.user.is_superuser:
facility = []
groups = []
users = FacilityUser.objects.all().order_by(
"last_name", "first_name")
else:
return HttpResponseNotFound(
_("Did not specify facility, group, nor user."))
# Query out the data: where?
if not form.data.get("topic_path"):
return HttpResponseNotFound(_("Must specify a topic path"))
# Query out the data: what?
computed_data = compute_data(
data_types=[form.data.get("xaxis"),
form.data.get("yaxis")],
who=users,
where=form.data.get("topic_path"),
language=language)
# Quickly add back in exercise meta-data (could potentially be used in future for other data too!)
ex_nodes = get_node_cache(language=language)["Exercise"]
exercises = []
for e in computed_data["exercises"]:
exercises.append({
"slug": e,
"full_name": ex_nodes[e]["display_name"],
"url": ex_nodes[e]["path"],
})
json_data = {
"data":
computed_data["data"],
"exercises":
exercises,
"videos":
computed_data["videos"],
"users":
dict(
zip([u.id for u in users], [
"%s, %s" % (u.last_name, u.first_name)
if u.last_name or u.first_name else u.username for u in users
])),
"groups":
dict(
zip(
[g.id for g in groups],
dict(zip(["id", "name"], [(g.id, g.name) for g in groups])),
)),
"facility":
None if not facility else {
"name": facility.name,
"id": facility.id,
}
}
if "facility_user" in request.session:
try:
# Log a "begin" and end here
user = request.session["facility_user"]
UserLog.begin_user_activity(user, activity_type="coachreport")
UserLog.update_user_activity(
user, activity_type="login"
) # to track active login time for teachers
UserLog.end_user_activity(user, activity_type="coachreport")
except ValidationError as e:
# Never report this error; don't want this logging to block other functionality.
logging.error(
"Failed to update Teacher userlog activity login: %s" % e)
# Now we have data, stream it back with a handler for date-times
return JsonResponse(json_data)
def tabular_view(request, report_type="exercise"):
"""Tabular view also gets data server-side."""
# important for setting the defaults for the coach nav bar
language = lcode_to_django_lang(request.language)
facility, group_id, context = coach_nav_context(request, "tabular")
# Define how students are ordered--used to be as efficient as possible.
student_ordering = ["last_name", "first_name", "username"]
# Get a list of topics (sorted) and groups
topics = [
get_node_cache("Topic", language=language).get(tid["id"])
for tid in get_knowledgemap_topics(language=language)
if report_type.title() in tid["contains"]
]
playlists = Playlist.all()
(groups, facilities,
ungrouped_available) = get_accessible_objects_from_logged_in_user(
request, facility=facility)
context.update(plotting_metadata_context(request, facility=facility))
context.update({
# For translators: the following two translations are nouns
"report_types": ({
"value": "exercise",
"name": _("exercise")
}, {
"value": "video",
"name": _("video")
}),
"request_report_type":
report_type,
"topics": [{
"id": t["id"],
"title": t["title"]
} for t in topics if t],
"playlists": [{
"id": p.id,
"title": p.title,
"tag": p.tag
} for p in playlists if p],
})
# get querystring info
topic_id = request.GET.get("topic", "")
playlist_id = request.GET.get("playlist", "")
# No valid data; just show generic
# Exactly one of topic_id or playlist_id should be present
if not ((topic_id or playlist_id) and not (topic_id and playlist_id)):
if playlists:
messages.add_message(request, WARNING,
_("Please select a playlist."))
elif topics:
messages.add_message(request, WARNING, _("Please select a topic."))
return context
playlist = (filter(lambda p: p.id == playlist_id, Playlist.all())
or [None])[0]
if group_id:
# Narrow by group
if group_id == control_panel_api_resources.UNGROUPED_KEY:
users = FacilityUser.objects.filter(group__isnull=True,
is_teacher=False)
if facility:
# filter only those ungrouped students for the facility
users = users.filter(facility=facility)
users = users.order_by(*student_ordering)
else:
# filter all ungroup students
users = FacilityUser.objects.filter(
group__isnull=True,
is_teacher=False).order_by(*student_ordering)
else:
users = FacilityUser.objects.filter(
group=group_id, is_teacher=False).order_by(*student_ordering)
elif facility:
# Narrow by facility
search_groups = [
groups_dict["groups"] for groups_dict in groups
if groups_dict["facility"] == facility.id
]
assert len(search_groups) <= 1, "Should only have one or zero matches."
# Return groups and ungrouped
search_groups = search_groups[
0] # make sure to include ungrouped students
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None, facility=facility),
is_teacher=False).order_by(*student_ordering)
else:
# Show all (including ungrouped)
search_groups = []
for groups_dict in groups:
search_groups += groups_dict["groups"]
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None),
is_teacher=False).order_by(*student_ordering)
# We have enough data to render over a group of students
# Get type-specific information
if report_type == "exercise":
# Fill in exercises
if topic_id:
exercises = get_topic_exercises(topic_id=topic_id)
elif playlist:
exercises = playlist.get_playlist_entries("Exercise",
language=language)
context["exercises"] = exercises
# More code, but much faster
exercise_names = [ex["id"] for ex in context["exercises"]]
# Get students
context["students"] = []
exlogs = ExerciseLog.objects \
.filter(user__in=users, exercise_id__in=exercise_names) \
.order_by(*["user__%s" % field for field in student_ordering]) \
.values("user__id", "struggling", "complete", "exercise_id")
exlogs = list(exlogs) # force the query to be evaluated
exlog_idx = 0
for user in users:
log_table = {}
while exlog_idx < len(
exlogs) and exlogs[exlog_idx]["user__id"] == user.id:
log_table[exlogs[exlog_idx]["exercise_id"]] = exlogs[exlog_idx]
exlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"exercise_logs": log_table,
})
elif report_type == "video":
# Fill in videos
if topic_id:
context["videos"] = get_topic_videos(topic_id=topic_id)
elif playlist:
context["videos"] = playlist.get_playlist_entries(
"Video", language=language)
# More code, but much faster
video_ids = [vid["id"] for vid in context["videos"]]
# Get students
context["students"] = []
vidlogs = VideoLog.objects \
.filter(user__in=users, video_id__in=video_ids) \
.order_by(*["user__%s" % field for field in student_ordering])\
.values("user__id", "complete", "video_id", "total_seconds_watched", "points")
vidlogs = list(vidlogs) # force the query to be executed now
vidlog_idx = 0
for user in users:
log_table = {}
while vidlog_idx < len(
vidlogs) and vidlogs[vidlog_idx]["user__id"] == user.id:
log_table[vidlogs[vidlog_idx]
["video_id"]] = vidlogs[vidlog_idx]
vidlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"video_logs": log_table,
})
else:
raise Http404(
_("Unknown report_type: %(report_type)s") %
{"report_type": report_type})
# Validate results by showing user messages.
if not users:
# 1. check group facility groups
if len(groups) > 0 and not groups[0]['groups']:
# 1. No groups available (for facility) and "no students" returned.
messages.add_message(
request, WARNING,
_("No learner accounts have been created for selected facility/group."
))
elif topic_id and playlist_id:
# 2. Both topic and playlist are selected.
messages.add_message(
request, WARNING,
_("Please select either a topic or a playlist above, but not both."
))
elif not topic_id and not playlist_id:
# 3. Group was selected, but data not queried because a topic or playlist was not selected.
if playlists:
# 4. No playlist was selected.
messages.add_message(request, WARNING,
_("Please select a playlist."))
elif topics:
# 5. No topic was selected.
messages.add_message(request, WARNING,
_("Please select a topic."))
else:
# 6. Everything specified, but no users fit the query.
messages.add_message(
request, WARNING,
_("No learner accounts in this group have been created."))
# End: Validate results by showing user messages.
log_coach_report_view(request)
return context
def tabular_view(request, facility, report_type="exercise"):
"""Tabular view also gets data server-side."""
# Define how students are ordered--used to be as efficient as possible.
student_ordering = ["last_name", "first_name", "username"]
# Get a list of topics (sorted) and groups
topics = [get_node_cache("Topic").get(tid) for tid in get_knowledgemap_topics()]
(groups, facilities) = get_accessible_objects_from_logged_in_user(request, facility=facility)
context = plotting_metadata_context(request, facility=facility)
context.update({
# For translators: the following two translations are nouns
"report_types": (_("exercise"), _("video")),
"request_report_type": report_type,
"topics": [{"id": t[0]["id"], "title": t[0]["title"]} for t in topics if t],
})
# get querystring info
topic_id = request.GET.get("topic", "")
# No valid data; just show generic
if not topic_id or not re.match("^[\w\-]+$", topic_id):
return context
group_id = request.GET.get("group", "")
if group_id:
# Narrow by group
users = FacilityUser.objects.filter(
group=group_id, is_teacher=False).order_by(*student_ordering)
elif facility:
# Narrow by facility
search_groups = [groups_dict["groups"] for groups_dict in groups if groups_dict["facility"] == facility.id]
assert len(search_groups) <= 1, "Should only have one or zero matches."
# Return groups and ungrouped
search_groups = search_groups[0] # make sure to include ungrouped students
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None, facility=facility), is_teacher=False).order_by(*student_ordering)
else:
# Show all (including ungrouped)
for groups_dict in groups:
search_groups += groups_dict["groups"]
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None), is_teacher=False).order_by(*student_ordering)
# We have enough data to render over a group of students
# Get type-specific information
if report_type == "exercise":
# Fill in exercises
exercises = get_topic_exercises(topic_id=topic_id)
exercises = sorted(exercises, key=lambda e: (e["h_position"], e["v_position"]))
context["exercises"] = exercises
# More code, but much faster
exercise_names = [ex["name"] for ex in context["exercises"]]
# Get students
context["students"] = []
exlogs = ExerciseLog.objects \
.filter(user__in=users, exercise_id__in=exercise_names) \
.order_by(*["user__%s" % field for field in student_ordering]) \
.values("user__id", "struggling", "complete", "exercise_id")
exlogs = list(exlogs) # force the query to be evaluated
exlog_idx = 0
for user in users:
log_table = {}
while exlog_idx < len(exlogs) and exlogs[exlog_idx]["user__id"] == user.id:
log_table[exlogs[exlog_idx]["exercise_id"]] = exlogs[exlog_idx]
exlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"exercise_logs": log_table,
})
elif report_type == "video":
# Fill in videos
context["videos"] = get_topic_videos(topic_id=topic_id)
# More code, but much faster
video_ids = [vid["id"] for vid in context["videos"]]
# Get students
context["students"] = []
vidlogs = VideoLog.objects \
.filter(user__in=users, video_id__in=video_ids) \
.order_by(*["user__%s" % field for field in student_ordering])\
.values("user__id", "complete", "video_id", "total_seconds_watched", "points")
vidlogs = list(vidlogs) # force the query to be executed now
vidlog_idx = 0
for user in users:
log_table = {}
while vidlog_idx < len(vidlogs) and vidlogs[vidlog_idx]["user__id"] == user.id:
log_table[vidlogs[vidlog_idx]["video_id"]] = vidlogs[vidlog_idx]
vidlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"video_logs": log_table,
})
else:
raise Http404(_("Unknown report_type: %(report_type)s") % {"report_type": report_type})
if "facility_user" in request.session:
try:
# Log a "begin" and end here
user = request.session["facility_user"]
UserLog.begin_user_activity(user, activity_type="coachreport")
UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers
UserLog.end_user_activity(user, activity_type="coachreport")
except ValidationError as e:
# Never report this error; don't want this logging to block other functionality.
logging.error("Failed to update Teacher userlog activity login: %s" % e)
return context
def student_view_context(request, xaxis="pct_mastery", yaxis="ex:attempts"):
"""
Context done separately, to be importable for similar pages.
"""
user = get_user_from_request(request=request)
if not user:
raise Http404("User not found.")
node_cache = get_node_cache()
topic_ids = get_knowledgemap_topics()
topic_ids = topic_ids + [ch["id"] for node in get_topic_tree()["children"] for ch in node["children"] if node["id"] != "math"]
topics = [node_cache["Topic"][id][0] for id in topic_ids]
user_id = user.id
exercise_logs = list(ExerciseLog.objects \
.filter(user=user) \
.values("exercise_id", "complete", "points", "attempts", "streak_progress", "struggling", "completion_timestamp"))
video_logs = list(VideoLog.objects \
.filter(user=user) \
.values("video_id", "complete", "total_seconds_watched", "points", "completion_timestamp"))
exercise_sparklines = dict()
stats = dict()
topic_exercises = dict()
topic_videos = dict()
exercises_by_topic = dict()
videos_by_topic = dict()
# Categorize every exercise log into a "midlevel" exercise
for elog in exercise_logs:
if not elog["exercise_id"] in node_cache["Exercise"]:
# Sometimes KA updates their topic tree and eliminates exercises;
# we also want to support 3rd party switching of trees arbitrarily.
logging.debug("Skip unknown exercise log for %s/%s" % (user_id, elog["exercise_id"]))
continue
parent_ids = [topic for ex in node_cache["Exercise"][elog["exercise_id"]] for topic in ex["ancestor_ids"]]
topic = set(parent_ids).intersection(set(topic_ids))
if not topic:
logging.error("Could not find a topic for exercise %s (parents=%s)" % (elog["exercise_id"], parent_ids))
continue
topic = topic.pop()
if not topic in topic_exercises:
topic_exercises[topic] = get_topic_exercises(path=node_cache["Topic"][topic][0]["path"])
exercises_by_topic[topic] = exercises_by_topic.get(topic, []) + [elog]
# Categorize every video log into a "midlevel" exercise.
for vlog in video_logs:
if not vlog["video_id"] in node_cache["Video"]:
# Sometimes KA updates their topic tree and eliminates videos;
# we also want to support 3rd party switching of trees arbitrarily.
logging.debug("Skip unknown video log for %s/%s" % (user_id, vlog["video_id"]))
continue
parent_ids = [topic for vid in node_cache["Video"][vlog["video_id"]] for topic in vid["ancestor_ids"]]
topic = set(parent_ids).intersection(set(topic_ids))
if not topic:
logging.error("Could not find a topic for video %s (parents=%s)" % (vlog["video_id"], parent_ids))
continue
topic = topic.pop()
if not topic in topic_videos:
topic_videos[topic] = get_topic_videos(path=node_cache["Topic"][topic][0]["path"])
videos_by_topic[topic] = videos_by_topic.get(topic, []) + [vlog]
# Now compute stats
for id in topic_ids:#set(topic_exercises.keys()).union(set(topic_videos.keys())):
n_exercises = len(topic_exercises.get(id, []))
n_videos = len(topic_videos.get(id, []))
exercises = exercises_by_topic.get(id, [])
videos = videos_by_topic.get(id, [])
n_exercises_touched = len(exercises)
n_videos_touched = len(videos)
exercise_sparklines[id] = [el["completion_timestamp"] for el in filter(lambda n: n["complete"], exercises)]
# total streak currently a pct, but expressed in max 100; convert to
# proportion (like other percentages here)
stats[id] = {
"ex:pct_mastery": 0 if not n_exercises_touched else sum([el["complete"] for el in exercises]) / float(n_exercises),
"ex:pct_started": 0 if not n_exercises_touched else n_exercises_touched / float(n_exercises),
"ex:average_points": 0 if not n_exercises_touched else sum([el["points"] for el in exercises]) / float(n_exercises_touched),
"ex:average_attempts": 0 if not n_exercises_touched else sum([el["attempts"] for el in exercises]) / float(n_exercises_touched),
"ex:average_streak": 0 if not n_exercises_touched else sum([el["streak_progress"] for el in exercises]) / float(n_exercises_touched) / 100.,
"ex:total_struggling": 0 if not n_exercises_touched else sum([el["struggling"] for el in exercises]),
"ex:last_completed": None if not n_exercises_touched else max_none([el["completion_timestamp"] or None for el in exercises]),
"vid:pct_started": 0 if not n_videos_touched else n_videos_touched / float(n_videos),
"vid:pct_completed": 0 if not n_videos_touched else sum([vl["complete"] for vl in videos]) / float(n_videos),
"vid:total_minutes": 0 if not n_videos_touched else sum([vl["total_seconds_watched"] for vl in videos]) / 60.,
"vid:average_points": 0. if not n_videos_touched else float(sum([vl["points"] for vl in videos]) / float(n_videos_touched)),
"vid:last_completed": None if not n_videos_touched else max_none([vl["completion_timestamp"] or None for vl in videos]),
}
context = plotting_metadata_context(request)
return {
"form": context["form"],
"groups": context["groups"],
"facilities": context["facilities"],
"student": user,
"topics": topics,
"exercises": topic_exercises,
"exercise_logs": exercises_by_topic,
"video_logs": videos_by_topic,
"exercise_sparklines": exercise_sparklines,
"no_data": not exercise_logs and not video_logs,
"stats": stats,
"stat_defs": [ # this order determines the order of display
{"key": "ex:pct_mastery", "title": _("% Mastery"), "type": "pct"},
{"key": "ex:pct_started", "title": _("% Started"), "type": "pct"},
{"key": "ex:average_points", "title": _("Average Points"), "type": "float"},
{"key": "ex:average_attempts", "title": _("Average Attempts"), "type": "float"},
{"key": "ex:average_streak", "title": _("Average Streak"), "type": "pct"},
{"key": "ex:total_struggling", "title": _("Struggling"), "type": "int"},
{"key": "ex:last_completed", "title": _("Last Completed"), "type": "date"},
{"key": "vid:pct_completed", "title": _("% Completed"), "type": "pct"},
{"key": "vid:pct_started", "title": _("% Started"), "type": "pct"},
{"key": "vid:total_minutes", "title": _("Average Minutes Watched"),"type": "float"},
{"key": "vid:average_points", "title": _("Average Points"), "type": "float"},
{"key": "vid:last_completed", "title": _("Last Completed"), "type": "date"},
]
}
def test_topic_availability(self):
for topic in get_node_cache("Topic").values():
any_available = bool(sum([v.get("available", False) for v in topic["children"]]))
self.assertEqual(topic["available"], any_available, "Make sure topic availability matches video availability when only videos are available.")
def generate_dubbed_video_mappings_from_csv(csv_data=None):
# This CSV file is in standard format: separated by ",", quoted by '"'
logging.info("Parsing csv file.")
reader = csv.reader(StringIO(csv_data))
# Build a two-level video map.
# First key: language name
# Second key: english youtube ID
# Value: corresponding youtube ID in the new language.
video_map = {}
# Loop through each row in the spreadsheet.
for row in reader:
# skip over the header rows
if row[0].strip() in ["", "UPDATED:"]:
continue
elif row[0] == "SERIAL":
# Read the header row.
header_row = [
v.lower() for v in row
] # lcase all header row values (including language names)
slug_idx = header_row.index("title id")
english_idx = header_row.index("english")
assert slug_idx != -1, "Video slug column header should be found."
assert english_idx != -1, "English video column header should be found."
else:
# Rows 6 and beyond are data.
assert len(row) == len(
header_row), "Values line length equals headers line length"
# Grab the slug and english video ID.
video_slug = row[slug_idx]
english_video_id = row[english_idx]
assert english_video_id, "English Video ID should not be empty"
assert video_slug, "Slug should not be empty"
# English video is the first video ID column,
# and following columns (until the end) are other languages.
# Loop through those columns and, if a video exists,
# add it to the dictionary.
for idx in range(english_idx, len(row)):
if not row[idx]: # make sure there's a dubbed video
continue
lang = header_row[idx]
if lang not in video_map: # add the first level if it doesn't exist
video_map[lang] = {}
dubbed_youtube_id = row[idx]
if english_video_id == dubbed_youtube_id and lang != "english":
logging.error(
"Removing entry for (%s, %s): dubbed and english youtube ID are the same."
% (lang, english_video_id))
#elif dubbed_youtube_id in video_map[lang].values():
# Talked to Bilal, and this is actually supposed to be OK. Would throw us for a loop!
# For now, just keep one.
#for key in video_map[lang].keys():
# if video_map[lang][key] == dubbed_youtube_id:
# del video_map[lang][key]
# break
#logging.error("Removing entry for (%s, %s): the same dubbed video ID is used in two places, and we can only keep one in our current system." % (lang, english_video_id))
else:
video_map[lang][english_video_id] = row[
idx] # add the corresponding video id for the video, in this language.
# Now, validate the mappings with our topic data
known_videos = get_node_cache("Video").keys()
missing_videos = set(known_videos) - set(video_map["english"].keys())
extra_videos = set(video_map["english"].keys()) - set(known_videos)
if missing_videos:
logging.warn(
"There are %d known videos not in the list of dubbed videos" %
len(missing_videos))
logging.warn(
"Adding missing English videos to English dubbed video map")
for video in missing_videos:
video_map["english"][video] = video
if extra_videos:
logging.warn(
"There are %d videos in the list of dubbed videos that we have never heard of."
% len(extra_videos))
return video_map
def clean_exercise_id(self):
"""
Make sure the exercise ID is found.
"""
if not self.cleaned_data.get("exercise_id", "") in get_node_cache('Exercise'):
raise forms.ValidationError(_("Exercise ID not recognized"))
def test_topic_availability(self):
for topic in get_node_cache("Topic").values():
if topic.get("kind") == "Topic":
any_available = bool(sum([get_node_cache("Topic").get(v, {}).get("available", False) for v in topic.get("children", [])]))
self.assertEqual(topic["available"], any_available, "Make sure topic availability matches child availability when any children are available.")
def search(request):
# Inputs
page = int(request.GET.get('page', 1))
query = request.GET.get('query')
category = request.GET.get('category')
max_results_per_category = request.GET.get('max_results', 25)
# Outputs
query_error = None
possible_matches = {}
hit_max = {}
node_kinds = {
"Topic": ["Topic"],
"Exercise": ["Exercise"],
"Content": ["Video", "Audio", "Document"],
}
if query is None:
query_error = _("Error: query not specified.")
# elif len(query) < 3:
# query_error = _("Error: query too short.")
else:
query = query.lower()
# search for topic, video or exercise with matching title
for node_type, node_dict in topic_tools.get_node_cache().iteritems():
if category and node_type != category:
# Skip categories that don't match (if specified)
continue
exact_match = filter(lambda node: node["kind"] in node_kinds[node_type] and node["title"].lower() == query, node_dict.values())[:1]
if exact_match:
# Redirect to an exact match
return HttpResponseRedirect(reverse('learn') + exact_match[0]['path'])
# For efficiency, don't do substring matches when we've got lots of results
match_generator = (node for node in node_dict.values()
if node["kind"] in node_kinds[node_type] and (query in node["title"].lower() or query in [x.lower() for x in node.get('keywords', [])] or
query in [x.lower() for x in node.get('tags', [])]))
# Only return max results
try:
possible_matches[node_type] = list(islice(match_generator, (page-1)*max_results_per_category, page*max_results_per_category))
except ValueError:
return HttpResponseNotFound("Page does not exist")
hit_max[node_type] = next(match_generator, False)
previous_params = request.GET.copy()
previous_params['page'] = page - 1
previous_url = "?" + previous_params.urlencode()
next_params = request.GET.copy()
next_params['page'] = page + 1
next_url = "?" + next_params.urlencode()
return {
'title': _("Search results for '%(query)s'") % {"query": (query if query else "")},
'query_error': query_error,
'results': possible_matches,
'hit_max': hit_max,
'more': any(hit_max.values()),
'page': page,
'previous_url': previous_url,
'next_url': next_url,
'query': query,
'max_results': max_results_per_category,
'category': category,
}
class StudentExerciseTest(BrowserActionMixins, FacilityMixins, KALiteBrowserTestCase):
"""
Test exercises.
"""
student_username = '******'
student_password = '******'
EXERCISE_SLUG = 'addition_1'
MIN_POINTS = get_node_cache("Exercise")[EXERCISE_SLUG]["basepoints"]
MAX_POINTS = 2 * MIN_POINTS
def setUp(self):
"""
Create a student, log the student in, and go to the exercise page.
"""
super(StudentExerciseTest, self).setUp()
self.facility_name = "fac"
self.facility = self.create_facility(name=self.facility_name)
self.student = self.create_student(username=self.student_username,
password=self.student_password,
facility=self.facility)
self.browser_login_student(self.student_username, self.student_password, facility_name=self.facility_name)
self.browse_to(self.live_server_url + reverse("learn") + get_node_cache("Exercise")[self.EXERCISE_SLUG]["path"])
self.nanswers = self.browser.execute_script('return window.ExerciseParams.STREAK_CORRECT_NEEDED;')
def browser_get_current_points(self):
"""
Check the total points a student has accumulated, from an exercise page.
"""
try:
points_regexp = r'\((?P<points>\w+) points\)'
points_text = self.browser.find_element_by_css_selector('.progress-points').text
points = re.match(points_regexp, points_text).group('points')
return points
except AttributeError:
return ""
def browser_submit_answer(self, answer):
"""
From an exercise page, insert an answer into the text box and submit.
"""
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_element_located((By.ID, 'solutionarea'))
)
self.browser.find_element_by_id('solutionarea').find_element_by_css_selector('input[type=text]').click()
self.browser_send_keys(unicode(answer))
self.browser.find_element_by_id('check-answer-button').click()
try:
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.visibility_of_element_located((By.ID, 'next-question-button'))
)
correct = self.browser.find_element_by_id('next-question-button').get_attribute("value")=="Correct! Next question..."
except TimeoutException:
correct = False
return correct
@unittest.skipIf(settings.RUNNING_IN_TRAVIS, "I CAN'T TAKE THIS ANYMORE!")
@unittest.skipIf(getattr(settings, 'CONFIG_PACKAGE', None), "Fails if settings.CONFIG_PACKAGE is set.")
def test_question_correct_points_are_added(self):
"""
Answer an exercise correctly
"""
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_element_located((By.CLASS_NAME, 'mord'))
)
numbers = self.browser.find_elements_by_css_selector("span[class=mord][style]")
answer = sum(int(num.text) for num in numbers)
correct = self.browser_submit_answer(answer)
self.assertTrue(correct, "answer was incorrect")
elog = ExerciseLog.objects.get(exercise_id=self.EXERCISE_SLUG, user=self.student)
self.assertEqual(elog.streak_progress, 100 / self.nanswers, "Streak progress should be 10%")
self.assertFalse(elog.struggling, "Student is not struggling.")
self.assertEqual(elog.attempts, 1, "Student should have 1 attempt.")
self.assertFalse(elog.complete, "Student should not have completed the exercise.")
self.assertEqual(elog.attempts_before_completion, None, "Student should not have a value for attempts_before_completion.")
@unittest.skipIf(settings.RUNNING_IN_TRAVIS, "I CAN'T TAKE THIS ANYMORE!")
def test_question_incorrect_false(self):
"""
Answer an exercise incorrectly.
"""
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_element_located((By.CLASS_NAME, 'mord'))
)
correct = self.browser_submit_answer(0)
self.assertFalse(correct, "answer was correct")
elog = ExerciseLog.objects.get(exercise_id=self.EXERCISE_SLUG, user=self.student)
self.assertEqual(elog.streak_progress, 0, "Streak progress should be 0%")
self.assertFalse(elog.struggling, "Student is not struggling.")
self.assertEqual(elog.attempts, 1, "Student should have 1 attempt.")
self.assertFalse(elog.complete, "Student should not have completed the exercise.")
self.assertEqual(elog.attempts_before_completion, None, "Student should not have a value for attempts_before_completion.")
@unittest.skipIf(settings.RUNNING_IN_TRAVIS, "I CAN'T TAKE THIS ANYMORE!")
def test_question_incorrect_button_text_changes(self):
"""
Answer an exercise incorrectly, and make sure button text changes.
"""
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_element_located((By.CLASS_NAME, 'mord'))
)
self.browser_submit_answer(0)
answer_button_text = self.browser.find_element_by_id("check-answer-button").get_attribute("value")
self.assertTrue(answer_button_text == "Check Answer", "Answer button changed on incorrect answer!")
# @unittest.skipIf(getattr(settings, 'CONFIG_PACKAGE', None), "Fails if settings.CONFIG_PACKAGE is set.")
@unittest.skipIf(settings.RUNNING_IN_TRAVIS, "I CAN'T TAKE THIS ANYMORE!")
@override_settings(CONFIG_PACKAGE=None)
def test_exercise_mastery(self):
"""
Answer an exercise til mastery
"""
for ai in range(1, 1 + self.nanswers):
# Hey future maintainer! The visibility_of_element_located
# requires that the element be ACTUALLY visible on the screen!
# so you can't just have the test spawn a teeny-bitty browser to
# the side while you have the world cup occupying a big part of your
# screen.
ui.WebDriverWait(self.browser, 10).until(
expected_conditions.visibility_of_element_located((By.CLASS_NAME, 'mord'))
)
numbers = self.browser.find_elements_by_css_selector("span[class=mord][style]")
answer = sum(int(num.text) for num in numbers)
correct = self.browser_submit_answer(answer)
self.assertTrue(correct, "answer was incorrect")
self.browser_send_keys(Keys.RETURN) # move on to next question.
# Now test the models
elog = ExerciseLog.objects.get(exercise_id=self.EXERCISE_SLUG, user=self.student)
self.assertEqual(elog.streak_progress, 100, "Streak progress should be 100%")
self.assertFalse(elog.struggling, "Student is not struggling.")
self.assertEqual(elog.attempts, self.nanswers, "Student should have %s attempts. Got %s" % (self.nanswers, elog.attempts))
self.assertTrue(elog.complete, "Student should have completed the exercise.")
self.assertEqual(elog.attempts_before_completion, self.nanswers, "Student should have %s attempts for completion." % self.nanswers)
def search(request):
# Inputs
page = int(request.GET.get('page', 1))
query = request.GET.get('query')
category = request.GET.get('category')
max_results_per_category = request.GET.get('max_results', 25)
# Outputs
query_error = None
possible_matches = {}
hit_max = {}
node_kinds = {
"Topic": ["Topic"],
"Exercise": ["Exercise"],
"Content": ["Video", "Audio", "Document"],
}
if query is None:
query_error = _("Error: query not specified.")
# elif len(query) < 3:
# query_error = _("Error: query too short.")
else:
query = query.lower()
# search for topic, video or exercise with matching title
for node_type, node_dict in topic_tools.get_node_cache().iteritems():
if category and node_type != category:
# Skip categories that don't match (if specified)
continue
exact_match = filter(lambda node: node["kind"] in node_kinds[node_type] and node["title"].lower() == query, node_dict.values())[:1]
if exact_match:
# Redirect to an exact match
return HttpResponseRedirect(reverse('learn') + exact_match[0]['path'])
# For efficiency, don't do substring matches when we've got lots of results
match_generator = (node for node in node_dict.values()
if node["kind"] in node_kinds[node_type] and (query in node["title"].lower() or query in [x.lower() for x in node.get('keywords', [])] or
query in [x.lower() for x in node.get('tags', [])]))
# Only return max results
try:
possible_matches[node_type] = list(islice(match_generator, (page-1)*max_results_per_category, page*max_results_per_category))
except ValueError:
return HttpResponseNotFound("Page does not exist")
hit_max[node_type] = next(match_generator, False)
previous_params = request.GET.copy()
previous_params['page'] = page - 1
previous_url = "?" + previous_params.urlencode()
next_params = request.GET.copy()
next_params['page'] = page + 1
next_url = "?" + next_params.urlencode()
return {
'title': _("Search results for '%(query)s'") % {"query": (query if query else "")},
'query_error': query_error,
'results': possible_matches,
'hit_max': hit_max,
'more': any(hit_max.values()),
'page': page,
'previous_url': previous_url,
'next_url': next_url,
'query': query,
'max_results': max_results_per_category,
'category': category,
}
def _get_user_usage_data(users, groups=None, period_start=None, period_end=None, group_id=None):
"""
Returns facility user data, within the given date range.
"""
groups = groups or set([user.group for user in users])
# compute period start and end
# Now compute stats, based on queried data
num_exercises = len(get_node_cache('Exercise'))
user_data = OrderedDict()
group_data = OrderedDict()
# Make queries efficiently
exercise_logs = ExerciseLog.objects.filter(user__in=users, complete=True)
video_logs = VideoLog.objects.filter(user__in=users)
login_logs = UserLogSummary.objects.filter(user__in=users)
# filter results
if period_start:
exercise_logs = exercise_logs.filter(completion_timestamp__gte=period_start)
video_logs = video_logs.filter(completion_timestamp__gte=period_start)
login_logs = login_logs.filter(start_datetime__gte=period_start)
if period_end:
exercise_logs = exercise_logs.filter(completion_timestamp__lte=period_end)
video_logs = video_logs.filter(completion_timestamp__lte=period_end)
login_logs = login_logs.filter(end_datetime__lte=period_end)
# Force results in a single query
exercise_logs = list(exercise_logs.values("exercise_id", "user__pk"))
video_logs = list(video_logs.values("video_id", "user__pk"))
login_logs = list(login_logs.values("activity_type", "total_seconds", "user__pk"))
for user in users:
user_data[user.pk] = OrderedDict()
user_data[user.pk]["id"] = user.pk
user_data[user.pk]["first_name"] = user.first_name
user_data[user.pk]["last_name"] = user.last_name
user_data[user.pk]["username"] = user.username
user_data[user.pk]["group"] = user.group
user_data[user.pk]["total_report_views"] = 0#report_stats["count__sum"] or 0
user_data[user.pk]["total_logins"] =0# login_stats["count__sum"] or 0
user_data[user.pk]["total_hours"] = 0#login_stats["total_seconds__sum"] or 0)/3600.
user_data[user.pk]["total_exercises"] = 0
user_data[user.pk]["pct_mastery"] = 0.
user_data[user.pk]["exercises_mastered"] = []
user_data[user.pk]["total_videos"] = 0
user_data[user.pk]["videos_watched"] = []
for elog in exercise_logs:
user_data[elog["user__pk"]]["total_exercises"] += 1
user_data[elog["user__pk"]]["pct_mastery"] += 1. / num_exercises
user_data[elog["user__pk"]]["exercises_mastered"].append(elog["exercise_id"])
for vlog in video_logs:
user_data[vlog["user__pk"]]["total_videos"] += 1
user_data[vlog["user__pk"]]["videos_watched"].append(vlog["video_id"])
for llog in login_logs:
if llog["activity_type"] == UserLog.get_activity_int("coachreport"):
user_data[llog["user__pk"]]["total_report_views"] += 1
elif llog["activity_type"] == UserLog.get_activity_int("login"):
user_data[llog["user__pk"]]["total_hours"] += (llog["total_seconds"]) / 3600.
user_data[llog["user__pk"]]["total_logins"] += 1
for group in list(groups) + [None]*(group_id==None or group_id=="Ungrouped"): # None for ungrouped, if no group_id passed.
group_pk = getattr(group, "pk", None)
group_name = getattr(group, "name", _("Ungrouped"))
group_data[group_pk] = {
"id": group_pk,
"name": group_name,
"total_logins": 0,
"total_hours": 0,
"total_users": 0,
"total_videos": 0,
"total_exercises": 0,
"pct_mastery": 0,
}
# Add group data. Allow a fake group "Ungrouped"
for user in users:
group_pk = getattr(user.group, "pk", None)
if group_pk not in group_data:
logging.error("User %s still in nonexistent group %s!" % (user.id, group_pk))
continue
group_data[group_pk]["total_users"] += 1
group_data[group_pk]["total_logins"] += user_data[user.pk]["total_logins"]
group_data[group_pk]["total_hours"] += user_data[user.pk]["total_hours"]
group_data[group_pk]["total_videos"] += user_data[user.pk]["total_videos"]
group_data[group_pk]["total_exercises"] += user_data[user.pk]["total_exercises"]
total_mastery_so_far = (group_data[group_pk]["pct_mastery"] * (group_data[group_pk]["total_users"] - 1) + user_data[user.pk]["pct_mastery"])
group_data[group_pk]["pct_mastery"] = total_mastery_so_far / group_data[group_pk]["total_users"]
if len(group_data) == 1 and group_data.has_key(None):
if not group_data[None]["total_users"]:
del group_data[None]
return (user_data, group_data)
def tabular_view(request, report_type="exercise"):
"""Tabular view also gets data server-side."""
# important for setting the defaults for the coach nav bar
language = lcode_to_django_lang(request.language)
facility, group_id, context = coach_nav_context(request, "tabular")
# Define how students are ordered--used to be as efficient as possible.
student_ordering = ["last_name", "first_name", "username"]
# Get a list of topics (sorted) and groups
topics = [get_node_cache("Topic", language=language).get(tid["id"]) for tid in get_knowledgemap_topics(language=language) if report_type.title() in tid["contains"]]
playlists = Playlist.all()
(groups, facilities, ungrouped_available) = get_accessible_objects_from_logged_in_user(request, facility=facility)
context.update(plotting_metadata_context(request, facility=facility))
context.update({
# For translators: the following two translations are nouns
"report_types": ({"value": "exercise", "name":_("exercise")}, {"value": "video", "name": _("video")}),
"request_report_type": report_type,
"topics": [{"id": t["id"], "title": t["title"]} for t in topics if t],
"playlists": [{"id": p.id, "title": p.title, "tag": p.tag} for p in playlists if p],
})
# get querystring info
topic_id = request.GET.get("topic", "")
playlist_id = request.GET.get("playlist", "")
# No valid data; just show generic
# Exactly one of topic_id or playlist_id should be present
if not ((topic_id or playlist_id) and not (topic_id and playlist_id)):
if playlists:
messages.add_message(request, WARNING, _("Please select a playlist."))
elif topics:
messages.add_message(request, WARNING, _("Please select a topic."))
return context
playlist = (filter(lambda p: p.id == playlist_id, Playlist.all()) or [None])[0]
if group_id:
# Narrow by group
if group_id == control_panel_api_resources.UNGROUPED_KEY:
users = FacilityUser.objects.filter(group__isnull=True, is_teacher=False)
if facility:
# filter only those ungrouped students for the facility
users = users.filter(facility=facility)
users = users.order_by(*student_ordering)
else:
# filter all ungroup students
users = FacilityUser.objects.filter(group__isnull=True, is_teacher=False).order_by(*student_ordering)
else:
users = FacilityUser.objects.filter(
group=group_id, is_teacher=False).order_by(*student_ordering)
elif facility:
# Narrow by facility
search_groups = [groups_dict["groups"] for groups_dict in groups if groups_dict["facility"] == facility.id]
assert len(search_groups) <= 1, "Should only have one or zero matches."
# Return groups and ungrouped
search_groups = search_groups[0] # make sure to include ungrouped students
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None, facility=facility), is_teacher=False).order_by(*student_ordering)
else:
# Show all (including ungrouped)
search_groups = []
for groups_dict in groups:
search_groups += groups_dict["groups"]
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None), is_teacher=False).order_by(*student_ordering)
# We have enough data to render over a group of students
# Get type-specific information
if report_type == "exercise":
# Fill in exercises
if topic_id:
exercises = get_topic_exercises(topic_id=topic_id)
elif playlist:
exercises = playlist.get_playlist_entries("Exercise", language=language)
context["exercises"] = exercises
# More code, but much faster
exercise_names = [ex["id"] for ex in context["exercises"]]
# Get students
context["students"] = []
exlogs = ExerciseLog.objects \
.filter(user__in=users, exercise_id__in=exercise_names) \
.order_by(*["user__%s" % field for field in student_ordering]) \
.values("user__id", "struggling", "complete", "exercise_id")
exlogs = list(exlogs) # force the query to be evaluated
exlog_idx = 0
for user in users:
log_table = {}
while exlog_idx < len(exlogs) and exlogs[exlog_idx]["user__id"] == user.id:
log_table[exlogs[exlog_idx]["exercise_id"]] = exlogs[exlog_idx]
exlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"exercise_logs": log_table,
})
elif report_type == "video":
# Fill in videos
if topic_id:
context["videos"] = get_topic_videos(topic_id=topic_id)
elif playlist:
context["videos"] = playlist.get_playlist_entries("Video", language=language)
# More code, but much faster
video_ids = [vid["id"] for vid in context["videos"]]
# Get students
context["students"] = []
vidlogs = VideoLog.objects \
.filter(user__in=users, video_id__in=video_ids) \
.order_by(*["user__%s" % field for field in student_ordering])\
.values("user__id", "complete", "video_id", "total_seconds_watched", "points")
vidlogs = list(vidlogs) # force the query to be executed now
vidlog_idx = 0
for user in users:
log_table = {}
while vidlog_idx < len(vidlogs) and vidlogs[vidlog_idx]["user__id"] == user.id:
log_table[vidlogs[vidlog_idx]["video_id"]] = vidlogs[vidlog_idx]
vidlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"video_logs": log_table,
})
else:
raise Http404(_("Unknown report_type: %(report_type)s") % {"report_type": report_type})
# Validate results by showing user messages.
if not users:
# 1. check group facility groups
if len(groups) > 0 and not groups[0]['groups']:
# 1. No groups available (for facility) and "no students" returned.
messages.add_message(request, WARNING,
_("No learner accounts have been created for selected facility/group."))
elif topic_id and playlist_id:
# 2. Both topic and playlist are selected.
messages.add_message(request, WARNING, _("Please select either a topic or a playlist above, but not both."))
elif not topic_id and not playlist_id:
# 3. Group was selected, but data not queried because a topic or playlist was not selected.
if playlists:
# 4. No playlist was selected.
messages.add_message(request, WARNING, _("Please select a playlist."))
elif topics:
# 5. No topic was selected.
messages.add_message(request, WARNING, _("Please select a topic."))
else:
# 6. Everything specified, but no users fit the query.
messages.add_message(request, WARNING, _("No learner accounts in this group have been created."))
# End: Validate results by showing user messages.
log_coach_report_view(request)
return context
def _get_user_usage_data(users, groups=None, period_start=None, period_end=None, group_id=None):
"""
Returns facility user data, within the given date range.
"""
groups = groups or set([user.group for user in users])
# compute period start and end
# Now compute stats, based on queried data
num_exercises = len(get_node_cache('Exercise'))
user_data = OrderedDict()
group_data = OrderedDict()
# Make queries efficiently
exercise_logs = ExerciseLog.objects.filter(user__in=users, complete=True)
video_logs = VideoLog.objects.filter(user__in=users, total_seconds_watched__gt=0)
login_logs = UserLogSummary.objects.filter(user__in=users)
# filter results
login_logs = login_logs.filter(total_seconds__gt=0)
if period_start:
exercise_logs = exercise_logs.filter(completion_timestamp__gte=period_start)
if period_end:
# MUST: Fix the midnight bug where period end covers up to the prior day only because
# period end is datetime(year, month, day, hour=0, minute=0), meaning midnight of previous day.
# Example:
# If period_end == '2014-12-01', we cannot include the records dated '2014-12-01 09:30'.
# So to fix this, we change it to '2014-12-01 23:59.999999'.
period_end = dateutil.parser.parse(period_end)
period_end = period_end + dateutil.relativedelta.relativedelta(days=+1, microseconds=-1)
exercise_logs = exercise_logs.filter(completion_timestamp__lte=period_end)
if period_start and period_end:
exercise_logs = exercise_logs.filter(Q(completion_timestamp__gte=period_start) &
Q(completion_timestamp__lte=period_end))
q1 = Q(completion_timestamp__isnull=False) & \
Q(completion_timestamp__gte=period_start) & \
Q(completion_timestamp__lte=period_end)
q2 = Q(completion_timestamp__isnull=True)
video_logs = video_logs.filter(q1 | q2)
login_q1 = Q(start_datetime__gte=period_start) & Q(start_datetime__lte=period_end) & \
Q(end_datetime__gte=period_start) & Q(end_datetime__lte=period_end)
login_logs = login_logs.filter(login_q1)
# Force results in a single query
exercise_logs = list(exercise_logs.values("exercise_id", "user__pk"))
video_logs = list(video_logs.values("video_id", "user__pk"))
login_logs = list(login_logs.values("activity_type", "total_seconds", "user__pk"))
for user in users:
user_data[user.pk] = OrderedDict()
user_data[user.pk]["id"] = user.pk
user_data[user.pk]["first_name"] = user.first_name
user_data[user.pk]["last_name"] = user.last_name
user_data[user.pk]["username"] = user.username
user_data[user.pk]["group"] = user.group
user_data[user.pk]["total_report_views"] = 0#report_stats["count__sum"] or 0
user_data[user.pk]["total_logins"] =0# login_stats["count__sum"] or 0
user_data[user.pk]["total_hours"] = 0#login_stats["total_seconds__sum"] or 0)/3600.
user_data[user.pk]["total_exercises"] = 0
user_data[user.pk]["pct_mastery"] = 0.
user_data[user.pk]["exercises_mastered"] = []
user_data[user.pk]["total_videos"] = 0
user_data[user.pk]["videos_watched"] = []
for elog in exercise_logs:
user_data[elog["user__pk"]]["total_exercises"] += 1
user_data[elog["user__pk"]]["pct_mastery"] += 1. / num_exercises
user_data[elog["user__pk"]]["exercises_mastered"].append(elog["exercise_id"])
for vlog in video_logs:
user_data[vlog["user__pk"]]["total_videos"] += 1
user_data[vlog["user__pk"]]["videos_watched"].append(vlog["video_id"])
for llog in login_logs:
if llog["activity_type"] == UserLog.get_activity_int("coachreport"):
user_data[llog["user__pk"]]["total_report_views"] += 1
elif llog["activity_type"] == UserLog.get_activity_int("login"):
user_data[llog["user__pk"]]["total_hours"] += (llog["total_seconds"]) / 3600.
user_data[llog["user__pk"]]["total_logins"] += 1
for group in list(groups) + [None]*(group_id==None or group_id=="Ungrouped"): # None for ungrouped, if no group_id passed.
group_pk = getattr(group, "pk", None)
group_name = getattr(group, "name", _("Ungrouped"))
group_data[group_pk] = {
"id": group_pk,
"name": group_name,
"total_logins": 0,
"total_hours": 0,
"total_users": 0,
"total_videos": 0,
"total_exercises": 0,
"pct_mastery": 0,
}
# Add group data. Allow a fake group "Ungrouped"
for user in users:
group_pk = getattr(user.group, "pk", None)
if group_pk not in group_data:
logging.error("User %s still in nonexistent group %s!" % (user.id, group_pk))
continue
group_data[group_pk]["total_users"] += 1
group_data[group_pk]["total_logins"] += user_data[user.pk]["total_logins"]
group_data[group_pk]["total_hours"] += user_data[user.pk]["total_hours"]
group_data[group_pk]["total_videos"] += user_data[user.pk]["total_videos"]
group_data[group_pk]["total_exercises"] += user_data[user.pk]["total_exercises"]
total_mastery_so_far = (group_data[group_pk]["pct_mastery"] * (group_data[group_pk]["total_users"] - 1) + user_data[user.pk]["pct_mastery"])
group_data[group_pk]["pct_mastery"] = total_mastery_so_far / group_data[group_pk]["total_users"]
if len(group_data) == 1 and group_data.has_key(None):
if not group_data[None]["total_users"]:
del group_data[None]
return (user_data, group_data)