def save(self, update_userlog=True, *args, **kwargs):
# To deal with backwards compatibility,
# check video_id, whether imported or not.
if not self.video_id:
assert kwargs.get("imported", False), "video_id better be set by internal code."
assert self.youtube_id, "If not video_id, you better have set youtube_id!"
self.video_id = i18n.get_video_id(self.youtube_id) or self.youtube_id # for unknown videos, default to the youtube_id
if not kwargs.get("imported", False):
self.full_clean()
# Compute learner status
already_complete = self.complete
self.complete = (self.points >= VideoLog.POINTS_PER_VIDEO)
if not already_complete and self.complete:
self.completion_timestamp = datetime.now()
# Tell logins that they are still active (ignoring validation failures).
# TODO(bcipolli): Could log video information in the future.
if update_userlog:
try:
UserLog.update_user_activity(self.user, activity_type="login", update_datetime=(self.completion_timestamp or datetime.now()), language=self.language)
except ValidationError as e:
logging.error("Failed to update userlog during video: %s" % e)
super(VideoLog, self).save(*args, **kwargs)
def handle(self, *args, **options):
# Get the CSV data, either from a recent cache_file
# or from the internet
cache_dir = settings.MEDIA_ROOT
cache_file = os.path.join(cache_dir, "dubbed_videos.csv")
if not options["force"] and os.path.exists(cache_file) and datediff(datetime.datetime.now(), datetime.datetime.fromtimestamp(os.path.getctime(cache_file)), units="days") <= 14.0:
# Use cached data to generate the video map
csv_data = open(cache_file, "r").read()
(video_map, _) = generate_dubbed_video_mappings(csv_data=csv_data)
else:
# Use cached data to generate the video map
(video_map, csv_data) = generate_dubbed_video_mappings()
try:
ensure_dir(cache_dir)
with open(cache_file, "w") as fp:
fp.write(csv_data)
except Exception as e:
logging.error("Failed to make a local cache of the CSV data: %s" % e)
# Now we've built the map. Save it.
out_file = DUBBED_VIDEOS_MAPPING_FILEPATH
ensure_dir(os.path.dirname(out_file))
logging.info("Saving data to %s" % out_file)
with open(out_file, "w") as fp:
json.dump(video_map, fp)
logging.info("Done.")
def generate_test_files():
"""Insert asterisks as translations in po files"""
# Open them up and insert asterisks for all empty msgstrs
logging.info("Generating test po files")
en_po_dir = os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/")
for po_file in glob.glob(os.path.join(en_po_dir, "*.po")):
msgid_pattern = re.compile(r'msgid \"(.*)\"\nmsgstr', re.S | re.M)
content = open(os.path.join(en_po_dir, po_file), 'r').read()
results = content.split("\n\n")
with open(os.path.join(en_po_dir, "tmp.po"), 'w') as temp_file:
# We know the first block is static, so just dump that.
temp_file.write(results[0])
# Now work through actual translations
for result in results[1:]:
try:
msgid = re.findall(msgid_pattern, result)[0]
temp_file.write("\n\n")
temp_file.write(result.replace("msgstr \"\"", "msgstr \"***%s***\"" % msgid))
except Exception as e:
logging.error("Failed to insert test string: %s\n\n%s\n\n" % (e, result))
# Once done replacing, rename temp file to overwrite original
os.rename(os.path.join(en_po_dir, "tmp.po"), os.path.join(en_po_dir, po_file))
(out, err, rc) = compile_po_files("en")
if err:
logging.debug("Error executing compilemessages: %s" % err)
def update_all_distributed_callback(request):
"""
"""
if request.method != "POST":
raise PermissionDenied("Only POST allowed to this URL endpoint.")
videos = json.loads(request.POST["video_logs"])
exercises = json.loads(request.POST["exercise_logs"])
user = FacilityUser.objects.get(id=request.POST["user_id"])
node_cache = get_node_cache()
# Save videos
n_videos_uploaded = 0
for video in videos:
video_id = video['video_id']
youtube_id = video['youtube_id']
# Only save video logs for videos that we recognize.
if video_id not in node_cache["Video"]:
logging.warn("Skipping unknown video %s" % video_id)
continue
try:
(vl, _) = VideoLog.get_or_initialize(user=user, video_id=video_id, youtube_id=youtube_id)
for key,val in video.iteritems():
setattr(vl, key, val)
logging.debug("Saving video log for %s: %s" % (video_id, vl))
vl.save()
n_videos_uploaded += 1
except KeyError: #
logging.error("Could not save video log for data with missing values: %s" % video)
except Exception as e:
error_message = "Unexpected error importing videos: %s" % e
return JsonResponseMessageError(error_message)
# Save exercises
n_exercises_uploaded = 0
for exercise in exercises:
# Only save video logs for videos that we recognize.
if exercise['exercise_id'] not in node_cache['Exercise']:
logging.warn("Skipping unknown video %s" % exercise['exercise_id'])
continue
try:
(el, _) = ExerciseLog.get_or_initialize(user=user, exercise_id=exercise["exercise_id"])
for key,val in exercise.iteritems():
setattr(el, key, val)
logging.debug("Saving exercise log for %s: %s" % (exercise['exercise_id'], el))
el.save()
n_exercises_uploaded += 1
except KeyError:
logging.error("Could not save exercise log for data with missing values: %s" % exercise)
except Exception as e:
error_message = "Unexpected error importing exercises: %s" % e
return JsonResponseMessageError(error_message)
return JsonResponse({"success": "Uploaded %d exercises and %d videos" % (n_exercises_uploaded, n_videos_uploaded)})
def test_get_exercise_load_status(self):
for path in get_exercise_paths():
logging.debug("Testing path : " + path)
self.browser.get(self.live_server_url + path)
error_list = self.browser.execute_script("return window.js_errors;")
if error_list:
logging.error("Found JS error(s) while loading path: " + path)
for e in error_list:
logging.error(e)
self.assertFalse(error_list)
def move_video_sizes_file(lang_code):
lang_pack_location = os.path.join(LOCALE_ROOT, lang_code)
filename = os.path.basename(REMOTE_VIDEO_SIZE_FILEPATH)
src_path = os.path.join(lang_pack_location, filename)
dest_path = REMOTE_VIDEO_SIZE_FILEPATH
# replace the old remote_video_size json
if not os.path.exists(src_path):
logging.error("Could not find videos sizes file (%s)" % src_path)
else:
logging.debug('Moving %s to %s' % (src_path, dest_path))
shutil.move(src_path, dest_path)
def logout(request):
if "facility_user" in request.session:
# Logout, ignore any errors.
try:
UserLog.end_user_activity(request.session["facility_user"], activity_type="login")
except ValidationError as e:
logging.error("Failed to end_user_activity upon logout: %s" % e)
del request.session["facility_user"]
auth_logout(request)
next = request.GET.get("next", reverse("homepage"))
if next[0] != "/":
next = "/"
return HttpResponseRedirect(next)
def get_file2lang_map(force=False):
"""Map from youtube_id to language code"""
global YT2LANG_MAP
if YT2LANG_MAP is None or force:
YT2LANG_MAP = {}
for lang_code, dic in get_dubbed_video_map().iteritems():
for dubbed_youtube_id in dic.values():
if dubbed_youtube_id in YT2LANG_MAP:
# Sanity check, but must be failsafe, since we don't control these data
if YT2LANG_MAP[dubbed_youtube_id] == lang_code:
logging.warn("Duplicate entry found in %s language map for dubbed video %s" % (lang_code, dubbed_youtube_id))
else:
logging.error("Conflicting entry found in language map for video %s; overwriting previous entry of %s to %s." % (dubbed_youtube_id, YT2LANG_MAP[dubbed_youtube_id], lang_code))
YT2LANG_MAP[dubbed_youtube_id] = lang_code
return YT2LANG_MAP
def build_translations(project_id=None, project_key=None):
"""Build latest translations into zip archive on CrowdIn."""
if not project_id:
project_id = settings.CROWDIN_PROJECT_ID
if not project_key:
project_key = settings.CROWDIN_PROJECT_KEY
logging.info("Requesting that CrowdIn build a fresh zip of our translations")
request_url = "http://api.crowdin.net/api/project/%s/export?key=%s" % (project_id, project_key)
try:
resp = requests.get(request_url)
resp.raise_for_status()
except Exception as e:
logging.error(e)
def move_dubbed_video_map(lang_code):
lang_pack_location = os.path.join(LOCALE_ROOT, lang_code)
dubbed_video_dir = os.path.join(lang_pack_location, "dubbed_videos")
dvm_filepath = os.path.join(dubbed_video_dir, os.path.basename(DUBBED_VIDEOS_MAPPING_FILEPATH))
if not os.path.exists(dvm_filepath):
logging.error("Could not find downloaded dubbed video filepath: %s" % dvm_filepath)
else:
logging.debug("Moving dubbed video map to %s" % DUBBED_VIDEOS_MAPPING_FILEPATH)
ensure_dir(os.path.dirname(DUBBED_VIDEOS_MAPPING_FILEPATH))
shutil.move(dvm_filepath, DUBBED_VIDEOS_MAPPING_FILEPATH)
logging.debug("Removing emtpy directory")
try:
shutil.rmtree(dubbed_video_dir)
except Exception as e:
logging.error("Error removing dubbed video directory (%s): %s" % (dubbed_video_dir, e))
def account_management(request, org_id=None):
# Only log 'coachreport' activity for students,
# (otherwise it's hard to compare teachers)
if "facility_user" in request.session and not request.session["facility_user"].is_teacher and reverse("login") not in request.META.get("HTTP_REFERER", ""):
try:
# Log a "begin" and end here
user = request.session["facility_user"]
UserLog.begin_user_activity(user, activity_type="coachreport")
UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers
UserLog.end_user_activity(user, activity_type="coachreport")
except ValidationError as e:
# Never report this error; don't want this logging to block other functionality.
logging.error("Failed to update student userlog activity: %s" % e)
return student_view_context(request)
def download_crowdin_metadata(project_id=None, project_key=None):
"""Return tuple in format (total_strings, total_translated, percent_translated)"""
if not project_id:
project_id = settings.CROWDIN_PROJECT_ID
if not project_key:
project_key = settings.CROWDIN_PROJECT_KEY
request_url = "http://api.crowdin.net/api/project/%s/status?key=%s&json=True" % (project_id, project_key)
try:
resp = requests.get(request_url)
resp.raise_for_status()
crowdin_meta_dict = json.loads(resp.content)
except Exception as e:
logging.error("Error getting crowdin metadata: %s" % e)
crowdin_meta_dict = {}
return crowdin_meta_dict
def record_ping(cls, id, ip):
"""
We received a failed request to create a session; record that 'ping' in our DB
"""
try:
# Create the log (if necessary), update, and save
# TODO: make a base class (in django_utils) that has get_or_initialize, and use that
# to shorten things here
(cur_device, _) = UnregisteredDevice.objects.get_or_create(id=id)
(cur_log, _) = cls.get_or_initialize(device=cur_device) # get is safe, because device is unique
cur_log.npings += 1
cur_log.last_ip = ip
cur_log.save()
except Exception as e:
# Never block functionality
logging.error("Error recording unregistered device ping: %s" % e)
def _get_installed_language_packs():
"""
On-disk method to show currently installed languages and meta data.
"""
# There's always English...
installed_language_packs = [{
'code': 'en',
'software_version': VERSION,
'language_pack_version': 0,
'percent_translated': 100,
'subtitle_count': 0,
'name': 'English',
'native_name': 'English',
}]
# Loop through locale folders
for locale_dir in settings.LOCALE_PATHS:
if not os.path.exists(locale_dir):
continue
# Loop through folders in each locale dir
for django_disk_code in os.listdir(locale_dir):
# Inside each folder, read from the JSON file - language name, % UI trans, version number
try:
# Get the metadata
metadata_filepath = os.path.join(locale_dir, django_disk_code, "%s_metadata.json" % lcode_to_ietf(django_disk_code))
lang_meta = softload_json(metadata_filepath, raises=True)
logging.debug("Found language pack %s" % (django_disk_code))
except Exception as e:
if isinstance(e, IOError) and e.errno == 2:
logging.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir))
else:
logging.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e))
continue
installed_language_packs.append(lang_meta)
sorted_list = sorted(installed_language_packs, key=lambda m: m['name'].lower())
return OrderedDict([(lcode_to_ietf(val["code"]), val) for val in sorted_list])
def setUp(self):
"""Create a browser to use for test cases. Try a bunch of different browsers; hopefully one of them works!"""
super(BrowserTestCase, self).setUp()
# Clear the session cache after ever test case, to keep things clean.
Session.objects.all().delete()
# Can use already launched browser.
if self.persistent_browser:
(self.browser,self.admin_user,self.admin_pass) = setup_test_env(persistent_browser=self.persistent_browser)
# Must create a new browser to use
else:
for browser_type in ["Firefox", "Chrome", "Ie", "Opera"]:
try:
(self.browser,self.admin_user,self.admin_pass) = setup_test_env(browser_type=browser_type)
break
except Exception as e:
logging.error("Could not create browser %s through selenium: %s" % (browser_type, e))
def _execute(self):
current_activity = "begin"
endtime = time.time() + (self.duration * 60.)
while True:
if time.time() >= endtime:
current_activity = "end"
# Prep and do the current activity
try:
start_clock_time = datetime.datetime.today()
start_time = time.time()
result=self.activity[current_activity]["method"](self.activity[current_activity]["args"])
self.return_list.append((
current_activity,
'%02d:%02d:%02d' % (start_clock_time.hour,start_clock_time.minute,start_clock_time.second),
round((time.time() - start_time),2),
))
except Exception as e:
if current_activity != "end":
raise
else:
logging.error("Error on end: %s" % e)
if current_activity == "end":
break
# Wait before the next activity
if "duration" in self.activity[current_activity]:
if self.verbosity >= 2:
print "(" + str(self.behavior_profile-24601) + ")" + "sleeping for ", self.activity[current_activity]["duration"]
time.sleep(self.activity[current_activity]["duration"])
# Choose the next activity
next_activity_random = round(self.random.random(),2)
for threshold, next_activity in self.activity[current_activity]["nextstep"]:
if threshold >= next_activity_random:
if self.verbosity >= 2:
print "(" + str(self.behavior_profile-24601) + ")" + str(next_activity_random), "next_activity =", next_activity
current_activity = next_activity
break
def set_cached_password(cls, user, raw_password):
assert user.id, "Your user must have an ID before calling this function."
if not cls.is_enabled():
# Must delete, to make sure we don't get out of sync.
cls.invalidate_cached_password(user=user)
else:
try:
# Set the cached password.
n_cached_iters = cls.iters_for_user_type(user)
# TODO(bcipolli) Migrate this to an extended django class
# that uses get_or_initialize
cached_password = get_object_or_None(cls, user=user) or cls(user=user)
cached_password.password = crypt(raw_password, iterations=n_cached_iters)
cached_password.save()
logging.debug("Set cached password for user=%s; iterations=%d" % (user.username, n_cached_iters))
except Exception as e:
# If we fail to create a cache item... just keep going--functionality
# can still move forward.
logging.error(e)
def move_exercises(lang_code):
lang_pack_location = os.path.join(LOCALE_ROOT, lang_code)
src_exercise_dir = os.path.join(lang_pack_location, "exercises")
dest_exercise_dir = get_localized_exercise_dirpath(lang_code, is_central_server=False)
if not os.path.exists(src_exercise_dir):
logging.warn("Could not find downloaded exercises; skipping: %s" % src_exercise_dir)
else:
# Move over one at a time, to combine with any other resources that were there before.
ensure_dir(dest_exercise_dir)
all_exercise_files = glob.glob(os.path.join(src_exercise_dir, "*.html"))
logging.info("Moving %d downloaded exercises to %s" % (len(all_exercise_files), dest_exercise_dir))
for exercise_file in all_exercise_files:
shutil.move(exercise_file, os.path.join(dest_exercise_dir, os.path.basename(exercise_file)))
logging.debug("Removing emtpy directory")
try:
shutil.rmtree(src_exercise_dir)
except Exception as e:
logging.error("Error removing dubbed video directory (%s): %s" % (src_exercise_dir, e))
def scrape_exercise(exercise_id, lang_code, force=False):
ietf_lang_code = lcode_to_ietf(lang_code)
exercise_dest_filepath = get_exercise_filepath(exercise_id, lang_code=lang_code)
exercise_localized_root = os.path.dirname(exercise_dest_filepath)
if os.path.exists(exercise_dest_filepath) and not force:
return
exercise_url = "https://es.khanacademy.org/khan-exercises/exercises/%s.html?lang=%s" % (exercise_id, ietf_lang_code)
logging.info("Retrieving exercise %s from %s" % (exercise_id, exercise_url))
try:
ensure_dir(exercise_localized_root)
resp = requests.get(exercise_url)
resp.raise_for_status()
with open(exercise_dest_filepath, "wb") as fp:
fp.write(resp.content)
except Exception as e:
logging.error("Failed to download %s: %s" % (exercise_url, e))
def save(self, update_userlog=True, *args, **kwargs):
if not kwargs.get("imported", False):
self.full_clean()
# Compute learner status
if self.attempts > 20 and not self.complete:
self.struggling = True
already_complete = self.complete
self.complete = (self.streak_progress >= 100)
if not already_complete and self.complete:
self.struggling = False
self.completion_timestamp = datetime.now()
self.attempts_before_completion = self.attempts
# Tell logins that they are still active (ignoring validation failures).
# TODO(bcipolli): Could log exercise information in the future.
if update_userlog:
try:
UserLog.update_user_activity(self.user, activity_type="login", update_datetime=(self.completion_timestamp or datetime.now()), language=self.language)
except ValidationError as e:
logging.error("Failed to update userlog during exercise: %s" % e)
super(ExerciseLog, self).save(*args, **kwargs)
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the distributed server.")
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
#
ensure_dir(settings.CONTENT_ROOT)
# Get list of videos
lang_code = lcode_to_ietf(options["lang_code"])
video_map = get_dubbed_video_map(lang_code) or {}
video_ids = options["video_ids"].split(",") if options["video_ids"] else None
video_ids = video_ids or ([vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None)
video_ids = video_ids or video_map.keys()
# Download the videos
for video_id in video_ids:
if video_id in video_map:
youtube_id = video_map[video_id]
elif video_id in video_map.values():
# Perhaps they sent in a youtube ID? We can handle that!
youtube_id = video_id
else:
logging.error("No mapping for video_id=%s; skipping" % video_id)
continue
try:
scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"])
#scrape_thumbnail(youtube_id=youtube_id)
logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"]))
except Exception as e:
logging.error("Failed to download video %s: %s" % (youtube_id, e))
logging.info("Process complete.")
def download_subtitle(youtube_id, lang_code, format="srt"):
"""
Return subtitles for YouTube ID in language specified. Return False if they do not exist. Update local JSON accordingly.
Note: srt map deals with amara, so uses lower-cased ietf codes (e.g. en-us)
"""
assert format == "srt", "We only support srt download at the moment."
# srt map deals with amara, so uses ietf codes (e.g. en-us)
api_info_map = softload_json(SRTS_JSON_FILEPATH, raises=True)
# get amara id
amara_code = api_info_map.get(youtube_id, {}).get("amara_code")
# make request
# Please see http://amara.readthedocs.org/en/latest/api.html
base_url = "https://amara.org/api2/partners/videos"
resp = make_request(AMARA_HEADERS, "%s/%s/languages/%s/subtitles/?format=srt" % (
base_url, amara_code, lang_code.lower(),
))
if isinstance(resp, basestring):
return resp
else:
# return the subtitle text, replacing empty subtitle lines with
# spaces to make the FLV player happy
try:
resp.encoding = "UTF-8"
response = (resp.text or u"") \
.replace("\n\n\n", "\n \n\n") \
.replace("\r\n\r\n\r\n", "\r\n \r\n\r\n")
except Exception as e:
logging.error(e)
response = "client-error"
return response
def execute(self, iterations=1):
if iterations < 1: iterations = 1
if hasattr(self, 'max_iterations'):
if iterations > self.max_iterations:
iterations = self.max_iterations
self.return_dict['iterations'] = iterations
self.return_dict['individual_elapsed'] = {}
self.return_dict['post_execute_info'] = {}
self.return_dict['exceptions'] = {}
for i in range(iterations):
self.return_dict['exceptions'][i+1] = []
start_time = time.time()
try:
self._execute()
self.return_dict['individual_elapsed'][i+1] = time.time() - start_time
except Exception as e:
self.return_dict['individual_elapsed'][i+1] = None
self.return_dict['exceptions'][i+1].append(e)
logging.error("Exception running execute: %s" % e)
try:
self.return_dict['post_execute_info'][i+1] = self._get_post_execute_info()
except Exception as e:
self.return_dict['post_execute_info'][i+1] = None
self.return_dict['exceptions'][i+1].append(e)
logging.error("Exception getting execute info: %s" % e)
mean = lambda vals: sum(vals)/float(len(vals)) if len(vals) else None
self.return_dict['average_elapsed'] = mean([v for v in self.return_dict['individual_elapsed'].values() if v is not None])
try:
self._teardown()
except Exception as e:
logging.error(e)
return self.return_dict
def download_srt_from_3rd_party(lang_codes=None, **kwargs):
"""Download subtitles specified by command line args"""
lang_codes = lang_codes or get_all_prepped_lang_codes()
bad_languages = {}
for lang_code in lang_codes:
lang_code = lcode_to_ietf(lang_code)
lang_code = get_supported_language_map(lang_code)['amara']
try:
lang_map_filepath = get_lang_map_filepath(lang_code)
if not os.path.exists(lang_map_filepath):
videos = {} # happens if an unknown set for subtitles.
else:
with open(lang_map_filepath, "r") as fp:
videos = json.load(fp)
except Exception as e:
error_msg = "Error in subtitles metadata file for %s: %s" % (lang_code, e)
logging.error(error_msg)
bad_languages[lang_code] = error_msg
continue
try:
download_if_criteria_met(videos, lang_code=lang_code, **kwargs)
except Exception as e:
error_msg = "Error downloading subtitles for %s: %s" % (lang_code, e)
logging.error(error_msg)
bad_languages[lang_code] = error_msg
continue
# now report final results
if bad_languages:
outstr = "Failed to download subtitles for the following languages: %s" % (bad_languages.keys())
outstr += "\n" + str(bad_languages)
logging.error(outstr)
def update_all_central_callback(request):
"""
Callback after authentication.
Parses out the request token verification.
Then finishes the request by getting an auth token.
"""
if not "ACCESS_TOKEN" in request.session:
finish_auth(request)
exercises = get_api_resource(request, "/api/v1/user/exercises")
videos = get_api_resource(request, "/api/v1/user/videos")
node_cache = get_node_cache()
# Collate videos
video_logs = []
for video in videos:
# Assume that KA videos are all english-language, not dubbed (for now)
video_id = youtube_id = video.get('video', {}).get('youtube_id', "")
# Only save videos with progress
if not video.get('seconds_watched', None):
continue
# Only save video logs for videos that we recognize.
if video_id not in node_cache["Video"]:
logging.warn("Skipping unknown video %s" % video_id)
continue
try:
video_logs.append({
"video_id": video_id,
"youtube_id": youtube_id,
"total_seconds_watched": video['seconds_watched'],
"points": VideoLog.calc_points(video['seconds_watched'], video['duration']),
"complete": video['completed'],
"completion_timestamp": convert_ka_date(video['last_watched']) if video['completed'] else None,
})
logging.debug("Got video log for %s: %s" % (video_id, video_logs[-1]))
except KeyError: #
logging.error("Could not save video log for data with missing values: %s" % video)
# Collate exercises
exercise_logs = []
for exercise in exercises:
# Only save exercises that have any progress.
if not exercise.get('last_done', None):
continue
# Only save video logs for videos that we recognize.
slug = exercise.get('exercise', "")
if slug not in node_cache['Exercise']:
logging.warn("Skipping unknown video %s" % slug)
continue
try:
completed = exercise['streak'] >= 10
basepoints = node_cache['Exercise'][slug][0]['basepoints']
exercise_logs.append({
"exercise_id": slug,
"streak_progress": min(100, 100 * exercise['streak']/10), # duplicates logic elsewhere
"attempts": exercise['total_done'],
"points": ExerciseLog.calc_points(basepoints, ncorrect=exercise['streak'], add_randomness=False), # no randomness when importing from KA
"complete": completed,
"attempts_before_completion": exercise['total_done'] if not exercise['practiced'] else None, #can't figure this out if they practiced after mastery.
"completion_timestamp": convert_ka_date(exercise['proficient_date']) if completed else None,
})
logging.debug("Got exercise log for %s: %s" % (slug, exercise_logs[-1]))
except KeyError:
logging.error("Could not save exercise log for data with missing values: %s" % exercise)
# POST the data back to the distributed server
try:
dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None
logging.debug("POST'ing to %s" % request.session["distributed_callback_url"])
response = requests.post(
request.session["distributed_callback_url"],
cookies={ "csrftoken": request.session["distributed_csrf_token"] },
data = {
"csrfmiddlewaretoken": request.session["distributed_csrf_token"],
"video_logs": json.dumps(video_logs, default=dthandler),
"exercise_logs": json.dumps(exercise_logs, default=dthandler),
"user_id": request.session["distributed_user_id"],
}
)
logging.debug("Response (%d): %s" % (response.status_code, response.content))
except requests.exceptions.ConnectionError as e:
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": "error",
"message": _("Could not connect to your KA Lite installation to share Khan Academy data."),
"message_id": "id_khanload",
}))
except Exception as e:
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": "error",
"message": _("Failure to send data to your KA Lite installation: %s") % e,
"message_id": "id_khanload",
}))
try:
json_response = json.loads(response.content)
if not isinstance(json_response, dict) or len(json_response) != 1:
# Could not validate the message is a single key-value pair
raise Exception(_("Unexpected response format from your KA Lite installation."))
message_type = json_response.keys()[0]
message = json_response.values()[0]
except ValueError as e:
message_type = "error"
message = unicode(e)
except Exception as e:
message_type = "error"
message = _("Loading json object: %s") % e
# If something broke on the distribute d server, we are SCREWED.
# For now, just show the error to users.
#
# Ultimately, we have a message, would like to share with the distributed server.
# if response.status_code != 200:
# return HttpResponseServerError(response.content)
return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], {
"message_type": message_type,
"message": message,
"message_id": "id_khanload",
}))
def generate_dubbed_video_mappings(download_url=None, csv_data=None):
"""
Function to do the heavy lifting in getting the dubbed videos map.
Could be moved into utils
"""
if not download_url:
download_url = SPREADSHEET_BASE_URL
params = {'key': SPREADSHEET_ID, 'gid': SPREADSHEET_GID, 'output': SPREADSHEET_EXPORT_FORMAT}
else:
params = {}
if not csv_data:
logging.info("Downloading dubbed video data from %s" % download_url)
response = requests.get(download_url, params=params)
if response.status_code != 200:
raise CommandError("Failed to download dubbed video CSV data: status=%s" % response.status)
csv_data = response.content
# This CSV file is in standard format: separated by ",", quoted by '"'
logging.info("Parsing csv file.")
reader = csv.reader(StringIO(csv_data))
# Build a two-level video map.
# First key: language name
# Second key: english youtube ID
# Value: corresponding youtube ID in the new language.
video_map = {}
row_num = -1
try:
# Loop through each row in the spreadsheet.
while (True):
row_num += 1
row = reader.next()
if row_num < 4:
# Rows 1-4 are crap.
continue
elif row_num == 4:
# Row 5 is the header row.
header_row = [v.lower() for v in row] # lcase all header row values (including language names)
slug_idx = header_row.index("titled id")
english_idx = header_row.index("english")
assert slug_idx != -1, "Video slug column header should be found."
assert english_idx != -1, "English video column header should be found."
else:
# Rows 6 and beyond are data.
assert len(row) == len(header_row), "Values line length equals headers line length"
# Grab the slug and english video ID.
video_slug = row[slug_idx]
english_video_id = row[english_idx]
assert english_video_id, "English Video ID should not be empty"
assert video_slug, "Slug should not be empty"
# English video is the first video ID column,
# and following columns (until the end) are other languages.
# Loop through those columns and, if a video exists,
# add it to the dictionary.
for idx in range(english_idx, len(row)):
if not row[idx]: # make sure there's a dubbed video
continue
lang = header_row[idx]
if lang not in video_map: # add the first level if it doesn't exist
video_map[lang] = {}
dubbed_youtube_id = row[idx]
if english_video_id == dubbed_youtube_id and lang != "english":
logging.error("Removing entry for (%s, %s): dubbed and english youtube ID are the same." % (lang, english_video_id))
#elif dubbed_youtube_id in video_map[lang].values():
# Talked to Bilal, and this is actually supposed to be OK. Would throw us for a loop!
# For now, just keep one.
#for key in video_map[lang].keys():
# if video_map[lang][key] == dubbed_youtube_id:
# del video_map[lang][key]
# break
#logging.error("Removing entry for (%s, %s): the same dubbed video ID is used in two places, and we can only keep one in our current system." % (lang, english_video_id))
else:
video_map[lang][english_video_id] = row[idx] # add the corresponding video id for the video, in this language.
except StopIteration:
# The loop ends when the CSV file hits the end and throws a StopIteration
pass
# Now, validate the mappings with our topic data
known_videos = get_node_cache("Video").keys()
missing_videos = set(known_videos) - set(video_map["english"].keys())
extra_videos = set(video_map["english"].keys()) - set(known_videos)
if missing_videos:
logging.warn("There are %d known videos not in the list of dubbed videos" % len(missing_videos))
logging.warn("Adding missing English videos to English dubbed video map")
for video in missing_videos:
video_map["english"][video] = video
if extra_videos:
logging.warn("There are %d videos in the list of dubbed videos that we have never heard of." % len(extra_videos))
return (video_map, csv_data)
def login(request, facility):
facility_id = facility and facility.id or None
facilities = list(Facility.objects.all())
# Fix for #1211: refresh cached facility info when it's free and relevant
refresh_session_facility_info(request, facility_count=len(facilities))
if request.method == 'POST':
# log out any Django user or facility user
logout(request)
username = request.POST.get("username", "")
password = request.POST.get("password", "")
# first try logging in as a Django user
user = authenticate(username=username, password=password)
if user:
auth_login(request, user)
return HttpResponseRedirect(request.next or reverse("easy_admin"))
# try logging in as a facility user
form = LoginForm(data=request.POST, request=request, initial={"facility": facility_id})
if form.is_valid():
user = form.get_user()
try:
UserLog.begin_user_activity(user, activity_type="login", language=request.language) # Success! Log the event (ignoring validation failures)
except ValidationError as e:
logging.error("Failed to begin_user_activity upon login: %s" % e)
request.session["facility_user"] = user
messages.success(request, _("You've been logged in! We hope you enjoy your time with KA Lite ") +
_("-- be sure to log out when you finish."))
# Send them back from whence they came
landing_page = form.cleaned_data["callback_url"]
if not landing_page:
# Just going back to the homepage? We can do better than that.
landing_page = reverse("coach_reports") if form.get_user().is_teacher else None
landing_page = landing_page or (reverse("account_management") if not package_selected("RPi") else reverse("homepage"))
return HttpResponseRedirect(form.non_field_errors() or request.next or landing_page)
else:
messages.error(
request,
_("There was an error logging you in. Please correct any errors listed below, and try again."),
)
else: # render the unbound login form
referer = urlparse.urlparse(request.META["HTTP_REFERER"]).path if request.META.get("HTTP_REFERER") else None
# never use the homepage as the referer
if referer in [reverse("homepage"), reverse("add_facility_student")]:
referer = None
form = LoginForm(initial={"facility": facility_id, "callback_url": referer})
return {
"form": form,
"facilities": facilities,
"sign_up_url": reverse("add_facility_student"),
}
def api_data(request, xaxis="", yaxis=""):
"""Request contains information about what data are requested (who, what, and how).
Response should be a JSON object
* data contains the data, structred by user and then datatype
* the rest of the data is metadata, useful for displaying detailed info about data.
"""
# Get the request form
form = get_data_form(request, xaxis=xaxis, yaxis=yaxis) # (data=request.REQUEST)
# Query out the data: who?
if form.data.get("user"):
facility = []
groups = []
users = [get_object_or_404(FacilityUser, id=form.data.get("user"))]
elif form.data.get("group"):
facility = []
groups = [get_object_or_404(FacilityGroup, id=form.data.get("group"))]
users = FacilityUser.objects.filter(group=form.data.get("group"), is_teacher=False).order_by("last_name", "first_name")
elif form.data.get("facility"):
facility = get_object_or_404(Facility, id=form.data.get("facility"))
groups = FacilityGroup.objects.filter(facility__in=[form.data.get("facility")])
users = FacilityUser.objects.filter(facility__in=[form.data.get("facility")], is_teacher=False).order_by("last_name", "first_name")
else:
return HttpResponseNotFound(_("Did not specify facility, group, nor user."))
# Query out the data: where?
if not form.data.get("topic_path"):
return HttpResponseNotFound(_("Must specify a topic path"))
# Query out the data: what?
computed_data = compute_data(data_types=[form.data.get("xaxis"), form.data.get("yaxis")], who=users, where=form.data.get("topic_path"))
json_data = {
"data": computed_data["data"],
"exercises": computed_data["exercises"],
"videos": computed_data["videos"],
"users": dict(zip([u.id for u in users],
["%s, %s" % (u.last_name, u.first_name) for u in users]
)),
"groups": dict(zip([g.id for g in groups],
dict(zip(["id", "name"], [(g.id, g.name) for g in groups])),
)),
"facility": None if not facility else {
"name": facility.name,
"id": facility.id,
}
}
if "facility_user" in request.session:
try:
# Log a "begin" and end here
user = request.session["facility_user"]
UserLog.begin_user_activity(user, activity_type="coachreport")
UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers
UserLog.end_user_activity(user, activity_type="coachreport")
except ValidationError as e:
# Never report this error; don't want this logging to block other functionality.
logging.error("Failed to update Teacher userlog activity login: %s" % e)
# Now we have data, stream it back with a handler for date-times
return JsonResponse(json_data)
def student_view_context(request, xaxis="pct_mastery", yaxis="ex:attempts"):
"""
Context done separately, to be importable for similar pages.
"""
user = get_user_from_request(request=request)
if not user:
raise Http404("User not found.")
node_cache = get_node_cache()
topic_ids = get_knowledgemap_topics()
topic_ids += [ch["id"] for node in get_topic_tree()["children"] for ch in node["children"] if node["id"] != "math"]
topics = [node_cache["Topic"][id][0] for id in topic_ids]
user_id = user.id
exercise_logs = list(ExerciseLog.objects \
.filter(user=user) \
.values("exercise_id", "complete", "points", "attempts", "streak_progress", "struggling", "completion_timestamp"))
video_logs = list(VideoLog.objects \
.filter(user=user) \
.values("video_id", "complete", "total_seconds_watched", "points", "completion_timestamp"))
exercise_sparklines = dict()
stats = dict()
topic_exercises = dict()
topic_videos = dict()
exercises_by_topic = dict()
videos_by_topic = dict()
# Categorize every exercise log into a "midlevel" exercise
for elog in exercise_logs:
if not elog["exercise_id"] in node_cache["Exercise"]:
# Sometimes KA updates their topic tree and eliminates exercises;
# we also want to support 3rd party switching of trees arbitrarily.
logging.debug("Skip unknown exercise log for %s/%s" % (user_id, elog["exercise_id"]))
continue
parent_ids = [topic for ex in node_cache["Exercise"][elog["exercise_id"]] for topic in ex["ancestor_ids"]]
topic = set(parent_ids).intersection(set(topic_ids))
if not topic:
logging.error("Could not find a topic for exercise %s (parents=%s)" % (elog["exercise_id"], parent_ids))
continue
topic = topic.pop()
if not topic in topic_exercises:
topic_exercises[topic] = get_topic_exercises(path=node_cache["Topic"][topic][0]["path"])
exercises_by_topic[topic] = exercises_by_topic.get(topic, []) + [elog]
# Categorize every video log into a "midlevel" exercise.
for vlog in video_logs:
if not vlog["video_id"] in node_cache["Video"]:
# Sometimes KA updates their topic tree and eliminates videos;
# we also want to support 3rd party switching of trees arbitrarily.
logging.debug("Skip unknown video log for %s/%s" % (user_id, vlog["video_id"]))
continue
parent_ids = [topic for vid in node_cache["Video"][vlog["video_id"]] for topic in vid["ancestor_ids"]]
topic = set(parent_ids).intersection(set(topic_ids))
if not topic:
logging.error("Could not find a topic for video %s (parents=%s)" % (vlog["video_id"], parent_ids))
continue
topic = topic.pop()
if not topic in topic_videos:
topic_videos[topic] = get_topic_videos(path=node_cache["Topic"][topic][0]["path"])
videos_by_topic[topic] = videos_by_topic.get(topic, []) + [vlog]
# Now compute stats
for id in topic_ids:#set(topic_exercises.keys()).union(set(topic_videos.keys())):
n_exercises = len(topic_exercises.get(id, []))
n_videos = len(topic_videos.get(id, []))
exercises = exercises_by_topic.get(id, [])
videos = videos_by_topic.get(id, [])
n_exercises_touched = len(exercises)
n_videos_touched = len(videos)
exercise_sparklines[id] = [el["completion_timestamp"] for el in filter(lambda n: n["complete"], exercises)]
# total streak currently a pct, but expressed in max 100; convert to
# proportion (like other percentages here)
stats[id] = {
"ex:pct_mastery": 0 if not n_exercises_touched else sum([el["complete"] for el in exercises]) / float(n_exercises),
"ex:pct_started": 0 if not n_exercises_touched else n_exercises_touched / float(n_exercises),
"ex:average_points": 0 if not n_exercises_touched else sum([el["points"] for el in exercises]) / float(n_exercises_touched),
"ex:average_attempts": 0 if not n_exercises_touched else sum([el["attempts"] for el in exercises]) / float(n_exercises_touched),
"ex:average_streak": 0 if not n_exercises_touched else sum([el["streak_progress"] for el in exercises]) / float(n_exercises_touched) / 100.,
"ex:total_struggling": 0 if not n_exercises_touched else sum([el["struggling"] for el in exercises]),
"ex:last_completed": None if not n_exercises_touched else max_none([el["completion_timestamp"] or None for el in exercises]),
"vid:pct_started": 0 if not n_videos_touched else n_videos_touched / float(n_videos),
"vid:pct_completed": 0 if not n_videos_touched else sum([vl["complete"] for vl in videos]) / float(n_videos),
"vid:total_minutes": 0 if not n_videos_touched else sum([vl["total_seconds_watched"] for vl in videos]) / 60.,
"vid:average_points": 0. if not n_videos_touched else float(sum([vl["points"] for vl in videos]) / float(n_videos_touched)),
"vid:last_completed": None if not n_videos_touched else max_none([vl["completion_timestamp"] or None for vl in videos]),
}
context = plotting_metadata_context(request)
return {
"form": context["form"],
"groups": context["groups"],
"facilities": context["facilities"],
"student": user,
"topics": topics,
"exercises": topic_exercises,
"exercise_logs": exercises_by_topic,
"video_logs": videos_by_topic,
"exercise_sparklines": exercise_sparklines,
"no_data": not exercise_logs and not video_logs,
"stats": stats,
"stat_defs": [ # this order determines the order of display
{"key": "ex:pct_mastery", "title": _("% Mastery"), "type": "pct"},
{"key": "ex:pct_started", "title": _("% Started"), "type": "pct"},
{"key": "ex:average_points", "title": _("Average Points"), "type": "float"},
{"key": "ex:average_attempts", "title": _("Average Attempts"), "type": "float"},
{"key": "ex:average_streak", "title": _("Average Streak"), "type": "pct"},
{"key": "ex:total_struggling", "title": _("Struggling"), "type": "int"},
{"key": "ex:last_completed", "title": _("Last Completed"), "type": "date"},
{"key": "vid:pct_completed", "title": _("% Completed"), "type": "pct"},
{"key": "vid:pct_started", "title": _("% Started"), "type": "pct"},
{"key": "vid:total_minutes", "title": _("Average Minutes Watched"),"type": "float"},
{"key": "vid:average_points", "title": _("Average Points"), "type": "float"},
{"key": "vid:last_completed", "title": _("Last Completed"), "type": "date"},
]
}
def tabular_view(request, facility, report_type="exercise"):
"""Tabular view also gets data server-side."""
# Define how students are ordered--used to be as efficient as possible.
student_ordering = ["last_name", "first_name", "username"]
# Get a list of topics (sorted) and groups
topics = [get_node_cache("Topic").get(tid) for tid in get_knowledgemap_topics()]
(groups, facilities) = get_accessible_objects_from_logged_in_user(request, facility=facility)
context = plotting_metadata_context(request, facility=facility)
context.update({
# For translators: the following two translations are nouns
"report_types": (_("exercise"), _("video")),
"request_report_type": report_type,
"topics": [{"id": t[0]["id"], "title": t[0]["title"]} for t in topics if t],
})
# get querystring info
topic_id = request.GET.get("topic", "")
# No valid data; just show generic
if not topic_id or not re.match("^[\w\-]+$", topic_id):
return context
group_id = request.GET.get("group", "")
if group_id:
# Narrow by group
users = FacilityUser.objects.filter(
group=group_id, is_teacher=False).order_by(*student_ordering)
elif facility:
# Narrow by facility
search_groups = [groups_dict["groups"] for groups_dict in groups if groups_dict["facility"] == facility.id]
assert len(search_groups) <= 1, "Should only have one or zero matches."
# Return groups and ungrouped
search_groups = search_groups[0] # make sure to include ungrouped students
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None, facility=facility), is_teacher=False).order_by(*student_ordering)
else:
# Show all (including ungrouped)
for groups_dict in groups:
search_groups += groups_dict["groups"]
users = FacilityUser.objects.filter(
Q(group__in=search_groups) | Q(group=None), is_teacher=False).order_by(*student_ordering)
# We have enough data to render over a group of students
# Get type-specific information
if report_type == "exercise":
# Fill in exercises
exercises = get_topic_exercises(topic_id=topic_id)
exercises = sorted(exercises, key=lambda e: (e["h_position"], e["v_position"]))
context["exercises"] = exercises
# More code, but much faster
exercise_names = [ex["name"] for ex in context["exercises"]]
# Get students
context["students"] = []
exlogs = ExerciseLog.objects \
.filter(user__in=users, exercise_id__in=exercise_names) \
.order_by(*["user__%s" % field for field in student_ordering]) \
.values("user__id", "struggling", "complete", "exercise_id")
exlogs = list(exlogs) # force the query to be evaluated
exlog_idx = 0
for user in users:
log_table = {}
while exlog_idx < len(exlogs) and exlogs[exlog_idx]["user__id"] == user.id:
log_table[exlogs[exlog_idx]["exercise_id"]] = exlogs[exlog_idx]
exlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"exercise_logs": log_table,
})
elif report_type == "video":
# Fill in videos
context["videos"] = get_topic_videos(topic_id=topic_id)
# More code, but much faster
video_ids = [vid["id"] for vid in context["videos"]]
# Get students
context["students"] = []
vidlogs = VideoLog.objects \
.filter(user__in=users, video_id__in=video_ids) \
.order_by(*["user__%s" % field for field in student_ordering])\
.values("user__id", "complete", "video_id", "total_seconds_watched", "points")
vidlogs = list(vidlogs) # force the query to be executed now
vidlog_idx = 0
for user in users:
log_table = {}
while vidlog_idx < len(vidlogs) and vidlogs[vidlog_idx]["user__id"] == user.id:
log_table[vidlogs[vidlog_idx]["video_id"]] = vidlogs[vidlog_idx]
vidlog_idx += 1
context["students"].append({ # this could be DRYer
"first_name": user.first_name,
"last_name": user.last_name,
"username": user.username,
"name": user.get_name(),
"id": user.id,
"video_logs": log_table,
})
else:
raise Http404(_("Unknown report_type: %(report_type)s") % {"report_type": report_type})
if "facility_user" in request.session:
try:
# Log a "begin" and end here
user = request.session["facility_user"]
UserLog.begin_user_activity(user, activity_type="coachreport")
UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers
UserLog.end_user_activity(user, activity_type="coachreport")
except ValidationError as e:
# Never report this error; don't want this logging to block other functionality.
logging.error("Failed to update Teacher userlog activity login: %s" % e)
return context
