def save(self, update_userlog=True, *args, **kwargs): # To deal with backwards compatibility, # check video_id, whether imported or not. if not self.video_id: assert kwargs.get("imported", False), "video_id better be set by internal code." assert self.youtube_id, "If not video_id, you better have set youtube_id!" self.video_id = i18n.get_video_id(self.youtube_id) or self.youtube_id # for unknown videos, default to the youtube_id if not kwargs.get("imported", False): self.full_clean() # Compute learner status already_complete = self.complete self.complete = (self.points >= VideoLog.POINTS_PER_VIDEO) if not already_complete and self.complete: self.completion_timestamp = datetime.now() # Tell logins that they are still active (ignoring validation failures). # TODO(bcipolli): Could log video information in the future. if update_userlog: try: UserLog.update_user_activity(self.user, activity_type="login", update_datetime=(self.completion_timestamp or datetime.now()), language=self.language) except ValidationError as e: logging.error("Failed to update userlog during video: %s" % e) super(VideoLog, self).save(*args, **kwargs)
def handle(self, *args, **options): # Get the CSV data, either from a recent cache_file # or from the internet cache_dir = settings.MEDIA_ROOT cache_file = os.path.join(cache_dir, "dubbed_videos.csv") if not options["force"] and os.path.exists(cache_file) and datediff(datetime.datetime.now(), datetime.datetime.fromtimestamp(os.path.getctime(cache_file)), units="days") <= 14.0: # Use cached data to generate the video map csv_data = open(cache_file, "r").read() (video_map, _) = generate_dubbed_video_mappings(csv_data=csv_data) else: # Use cached data to generate the video map (video_map, csv_data) = generate_dubbed_video_mappings() try: ensure_dir(cache_dir) with open(cache_file, "w") as fp: fp.write(csv_data) except Exception as e: logging.error("Failed to make a local cache of the CSV data: %s" % e) # Now we've built the map. Save it. out_file = DUBBED_VIDEOS_MAPPING_FILEPATH ensure_dir(os.path.dirname(out_file)) logging.info("Saving data to %s" % out_file) with open(out_file, "w") as fp: json.dump(video_map, fp) logging.info("Done.")
def generate_test_files(): """Insert asterisks as translations in po files""" # Open them up and insert asterisks for all empty msgstrs logging.info("Generating test po files") en_po_dir = os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/") for po_file in glob.glob(os.path.join(en_po_dir, "*.po")): msgid_pattern = re.compile(r'msgid \"(.*)\"\nmsgstr', re.S | re.M) content = open(os.path.join(en_po_dir, po_file), 'r').read() results = content.split("\n\n") with open(os.path.join(en_po_dir, "tmp.po"), 'w') as temp_file: # We know the first block is static, so just dump that. temp_file.write(results[0]) # Now work through actual translations for result in results[1:]: try: msgid = re.findall(msgid_pattern, result)[0] temp_file.write("\n\n") temp_file.write(result.replace("msgstr \"\"", "msgstr \"***%s***\"" % msgid)) except Exception as e: logging.error("Failed to insert test string: %s\n\n%s\n\n" % (e, result)) # Once done replacing, rename temp file to overwrite original os.rename(os.path.join(en_po_dir, "tmp.po"), os.path.join(en_po_dir, po_file)) (out, err, rc) = compile_po_files("en") if err: logging.debug("Error executing compilemessages: %s" % err)
def update_all_distributed_callback(request): """ """ if request.method != "POST": raise PermissionDenied("Only POST allowed to this URL endpoint.") videos = json.loads(request.POST["video_logs"]) exercises = json.loads(request.POST["exercise_logs"]) user = FacilityUser.objects.get(id=request.POST["user_id"]) node_cache = get_node_cache() # Save videos n_videos_uploaded = 0 for video in videos: video_id = video['video_id'] youtube_id = video['youtube_id'] # Only save video logs for videos that we recognize. if video_id not in node_cache["Video"]: logging.warn("Skipping unknown video %s" % video_id) continue try: (vl, _) = VideoLog.get_or_initialize(user=user, video_id=video_id, youtube_id=youtube_id) for key,val in video.iteritems(): setattr(vl, key, val) logging.debug("Saving video log for %s: %s" % (video_id, vl)) vl.save() n_videos_uploaded += 1 except KeyError: # logging.error("Could not save video log for data with missing values: %s" % video) except Exception as e: error_message = "Unexpected error importing videos: %s" % e return JsonResponseMessageError(error_message) # Save exercises n_exercises_uploaded = 0 for exercise in exercises: # Only save video logs for videos that we recognize. if exercise['exercise_id'] not in node_cache['Exercise']: logging.warn("Skipping unknown video %s" % exercise['exercise_id']) continue try: (el, _) = ExerciseLog.get_or_initialize(user=user, exercise_id=exercise["exercise_id"]) for key,val in exercise.iteritems(): setattr(el, key, val) logging.debug("Saving exercise log for %s: %s" % (exercise['exercise_id'], el)) el.save() n_exercises_uploaded += 1 except KeyError: logging.error("Could not save exercise log for data with missing values: %s" % exercise) except Exception as e: error_message = "Unexpected error importing exercises: %s" % e return JsonResponseMessageError(error_message) return JsonResponse({"success": "Uploaded %d exercises and %d videos" % (n_exercises_uploaded, n_videos_uploaded)})
def test_get_exercise_load_status(self): for path in get_exercise_paths(): logging.debug("Testing path : " + path) self.browser.get(self.live_server_url + path) error_list = self.browser.execute_script("return window.js_errors;") if error_list: logging.error("Found JS error(s) while loading path: " + path) for e in error_list: logging.error(e) self.assertFalse(error_list)
def move_video_sizes_file(lang_code): lang_pack_location = os.path.join(LOCALE_ROOT, lang_code) filename = os.path.basename(REMOTE_VIDEO_SIZE_FILEPATH) src_path = os.path.join(lang_pack_location, filename) dest_path = REMOTE_VIDEO_SIZE_FILEPATH # replace the old remote_video_size json if not os.path.exists(src_path): logging.error("Could not find videos sizes file (%s)" % src_path) else: logging.debug('Moving %s to %s' % (src_path, dest_path)) shutil.move(src_path, dest_path)
def logout(request): if "facility_user" in request.session: # Logout, ignore any errors. try: UserLog.end_user_activity(request.session["facility_user"], activity_type="login") except ValidationError as e: logging.error("Failed to end_user_activity upon logout: %s" % e) del request.session["facility_user"] auth_logout(request) next = request.GET.get("next", reverse("homepage")) if next[0] != "/": next = "/" return HttpResponseRedirect(next)
def get_file2lang_map(force=False): """Map from youtube_id to language code""" global YT2LANG_MAP if YT2LANG_MAP is None or force: YT2LANG_MAP = {} for lang_code, dic in get_dubbed_video_map().iteritems(): for dubbed_youtube_id in dic.values(): if dubbed_youtube_id in YT2LANG_MAP: # Sanity check, but must be failsafe, since we don't control these data if YT2LANG_MAP[dubbed_youtube_id] == lang_code: logging.warn("Duplicate entry found in %s language map for dubbed video %s" % (lang_code, dubbed_youtube_id)) else: logging.error("Conflicting entry found in language map for video %s; overwriting previous entry of %s to %s." % (dubbed_youtube_id, YT2LANG_MAP[dubbed_youtube_id], lang_code)) YT2LANG_MAP[dubbed_youtube_id] = lang_code return YT2LANG_MAP
def build_translations(project_id=None, project_key=None): """Build latest translations into zip archive on CrowdIn.""" if not project_id: project_id = settings.CROWDIN_PROJECT_ID if not project_key: project_key = settings.CROWDIN_PROJECT_KEY logging.info("Requesting that CrowdIn build a fresh zip of our translations") request_url = "http://api.crowdin.net/api/project/%s/export?key=%s" % (project_id, project_key) try: resp = requests.get(request_url) resp.raise_for_status() except Exception as e: logging.error(e)
def move_dubbed_video_map(lang_code): lang_pack_location = os.path.join(LOCALE_ROOT, lang_code) dubbed_video_dir = os.path.join(lang_pack_location, "dubbed_videos") dvm_filepath = os.path.join(dubbed_video_dir, os.path.basename(DUBBED_VIDEOS_MAPPING_FILEPATH)) if not os.path.exists(dvm_filepath): logging.error("Could not find downloaded dubbed video filepath: %s" % dvm_filepath) else: logging.debug("Moving dubbed video map to %s" % DUBBED_VIDEOS_MAPPING_FILEPATH) ensure_dir(os.path.dirname(DUBBED_VIDEOS_MAPPING_FILEPATH)) shutil.move(dvm_filepath, DUBBED_VIDEOS_MAPPING_FILEPATH) logging.debug("Removing emtpy directory") try: shutil.rmtree(dubbed_video_dir) except Exception as e: logging.error("Error removing dubbed video directory (%s): %s" % (dubbed_video_dir, e))
def account_management(request, org_id=None): # Only log 'coachreport' activity for students, # (otherwise it's hard to compare teachers) if "facility_user" in request.session and not request.session["facility_user"].is_teacher and reverse("login") not in request.META.get("HTTP_REFERER", ""): try: # Log a "begin" and end here user = request.session["facility_user"] UserLog.begin_user_activity(user, activity_type="coachreport") UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers UserLog.end_user_activity(user, activity_type="coachreport") except ValidationError as e: # Never report this error; don't want this logging to block other functionality. logging.error("Failed to update student userlog activity: %s" % e) return student_view_context(request)
def download_crowdin_metadata(project_id=None, project_key=None): """Return tuple in format (total_strings, total_translated, percent_translated)""" if not project_id: project_id = settings.CROWDIN_PROJECT_ID if not project_key: project_key = settings.CROWDIN_PROJECT_KEY request_url = "http://api.crowdin.net/api/project/%s/status?key=%s&json=True" % (project_id, project_key) try: resp = requests.get(request_url) resp.raise_for_status() crowdin_meta_dict = json.loads(resp.content) except Exception as e: logging.error("Error getting crowdin metadata: %s" % e) crowdin_meta_dict = {} return crowdin_meta_dict
def record_ping(cls, id, ip): """ We received a failed request to create a session; record that 'ping' in our DB """ try: # Create the log (if necessary), update, and save # TODO: make a base class (in django_utils) that has get_or_initialize, and use that # to shorten things here (cur_device, _) = UnregisteredDevice.objects.get_or_create(id=id) (cur_log, _) = cls.get_or_initialize(device=cur_device) # get is safe, because device is unique cur_log.npings += 1 cur_log.last_ip = ip cur_log.save() except Exception as e: # Never block functionality logging.error("Error recording unregistered device ping: %s" % e)
def _get_installed_language_packs(): """ On-disk method to show currently installed languages and meta data. """ # There's always English... installed_language_packs = [{ 'code': 'en', 'software_version': VERSION, 'language_pack_version': 0, 'percent_translated': 100, 'subtitle_count': 0, 'name': 'English', 'native_name': 'English', }] # Loop through locale folders for locale_dir in settings.LOCALE_PATHS: if not os.path.exists(locale_dir): continue # Loop through folders in each locale dir for django_disk_code in os.listdir(locale_dir): # Inside each folder, read from the JSON file - language name, % UI trans, version number try: # Get the metadata metadata_filepath = os.path.join(locale_dir, django_disk_code, "%s_metadata.json" % lcode_to_ietf(django_disk_code)) lang_meta = softload_json(metadata_filepath, raises=True) logging.debug("Found language pack %s" % (django_disk_code)) except Exception as e: if isinstance(e, IOError) and e.errno == 2: logging.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir)) else: logging.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e)) continue installed_language_packs.append(lang_meta) sorted_list = sorted(installed_language_packs, key=lambda m: m['name'].lower()) return OrderedDict([(lcode_to_ietf(val["code"]), val) for val in sorted_list])
def setUp(self): """Create a browser to use for test cases. Try a bunch of different browsers; hopefully one of them works!""" super(BrowserTestCase, self).setUp() # Clear the session cache after ever test case, to keep things clean. Session.objects.all().delete() # Can use already launched browser. if self.persistent_browser: (self.browser,self.admin_user,self.admin_pass) = setup_test_env(persistent_browser=self.persistent_browser) # Must create a new browser to use else: for browser_type in ["Firefox", "Chrome", "Ie", "Opera"]: try: (self.browser,self.admin_user,self.admin_pass) = setup_test_env(browser_type=browser_type) break except Exception as e: logging.error("Could not create browser %s through selenium: %s" % (browser_type, e))
def _execute(self): current_activity = "begin" endtime = time.time() + (self.duration * 60.) while True: if time.time() >= endtime: current_activity = "end" # Prep and do the current activity try: start_clock_time = datetime.datetime.today() start_time = time.time() result=self.activity[current_activity]["method"](self.activity[current_activity]["args"]) self.return_list.append(( current_activity, '%02d:%02d:%02d' % (start_clock_time.hour,start_clock_time.minute,start_clock_time.second), round((time.time() - start_time),2), )) except Exception as e: if current_activity != "end": raise else: logging.error("Error on end: %s" % e) if current_activity == "end": break # Wait before the next activity if "duration" in self.activity[current_activity]: if self.verbosity >= 2: print "(" + str(self.behavior_profile-24601) + ")" + "sleeping for ", self.activity[current_activity]["duration"] time.sleep(self.activity[current_activity]["duration"]) # Choose the next activity next_activity_random = round(self.random.random(),2) for threshold, next_activity in self.activity[current_activity]["nextstep"]: if threshold >= next_activity_random: if self.verbosity >= 2: print "(" + str(self.behavior_profile-24601) + ")" + str(next_activity_random), "next_activity =", next_activity current_activity = next_activity break
def set_cached_password(cls, user, raw_password): assert user.id, "Your user must have an ID before calling this function." if not cls.is_enabled(): # Must delete, to make sure we don't get out of sync. cls.invalidate_cached_password(user=user) else: try: # Set the cached password. n_cached_iters = cls.iters_for_user_type(user) # TODO(bcipolli) Migrate this to an extended django class # that uses get_or_initialize cached_password = get_object_or_None(cls, user=user) or cls(user=user) cached_password.password = crypt(raw_password, iterations=n_cached_iters) cached_password.save() logging.debug("Set cached password for user=%s; iterations=%d" % (user.username, n_cached_iters)) except Exception as e: # If we fail to create a cache item... just keep going--functionality # can still move forward. logging.error(e)
def move_exercises(lang_code): lang_pack_location = os.path.join(LOCALE_ROOT, lang_code) src_exercise_dir = os.path.join(lang_pack_location, "exercises") dest_exercise_dir = get_localized_exercise_dirpath(lang_code, is_central_server=False) if not os.path.exists(src_exercise_dir): logging.warn("Could not find downloaded exercises; skipping: %s" % src_exercise_dir) else: # Move over one at a time, to combine with any other resources that were there before. ensure_dir(dest_exercise_dir) all_exercise_files = glob.glob(os.path.join(src_exercise_dir, "*.html")) logging.info("Moving %d downloaded exercises to %s" % (len(all_exercise_files), dest_exercise_dir)) for exercise_file in all_exercise_files: shutil.move(exercise_file, os.path.join(dest_exercise_dir, os.path.basename(exercise_file))) logging.debug("Removing emtpy directory") try: shutil.rmtree(src_exercise_dir) except Exception as e: logging.error("Error removing dubbed video directory (%s): %s" % (src_exercise_dir, e))
def scrape_exercise(exercise_id, lang_code, force=False): ietf_lang_code = lcode_to_ietf(lang_code) exercise_dest_filepath = get_exercise_filepath(exercise_id, lang_code=lang_code) exercise_localized_root = os.path.dirname(exercise_dest_filepath) if os.path.exists(exercise_dest_filepath) and not force: return exercise_url = "https://es.khanacademy.org/khan-exercises/exercises/%s.html?lang=%s" % (exercise_id, ietf_lang_code) logging.info("Retrieving exercise %s from %s" % (exercise_id, exercise_url)) try: ensure_dir(exercise_localized_root) resp = requests.get(exercise_url) resp.raise_for_status() with open(exercise_dest_filepath, "wb") as fp: fp.write(resp.content) except Exception as e: logging.error("Failed to download %s: %s" % (exercise_url, e))
def save(self, update_userlog=True, *args, **kwargs): if not kwargs.get("imported", False): self.full_clean() # Compute learner status if self.attempts > 20 and not self.complete: self.struggling = True already_complete = self.complete self.complete = (self.streak_progress >= 100) if not already_complete and self.complete: self.struggling = False self.completion_timestamp = datetime.now() self.attempts_before_completion = self.attempts # Tell logins that they are still active (ignoring validation failures). # TODO(bcipolli): Could log exercise information in the future. if update_userlog: try: UserLog.update_user_activity(self.user, activity_type="login", update_datetime=(self.completion_timestamp or datetime.now()), language=self.language) except ValidationError as e: logging.error("Failed to update userlog during exercise: %s" % e) super(ExerciseLog, self).save(*args, **kwargs)
def handle(self, *args, **options): if settings.CENTRAL_SERVER: raise CommandError("This must only be run on the distributed server.") if not options["lang_code"]: raise CommandError("You must specify a language code.") # ensure_dir(settings.CONTENT_ROOT) # Get list of videos lang_code = lcode_to_ietf(options["lang_code"]) video_map = get_dubbed_video_map(lang_code) or {} video_ids = options["video_ids"].split(",") if options["video_ids"] else None video_ids = video_ids or ([vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None) video_ids = video_ids or video_map.keys() # Download the videos for video_id in video_ids: if video_id in video_map: youtube_id = video_map[video_id] elif video_id in video_map.values(): # Perhaps they sent in a youtube ID? We can handle that! youtube_id = video_id else: logging.error("No mapping for video_id=%s; skipping" % video_id) continue try: scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"]) #scrape_thumbnail(youtube_id=youtube_id) logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"])) except Exception as e: logging.error("Failed to download video %s: %s" % (youtube_id, e)) logging.info("Process complete.")
def download_subtitle(youtube_id, lang_code, format="srt"): """ Return subtitles for YouTube ID in language specified. Return False if they do not exist. Update local JSON accordingly. Note: srt map deals with amara, so uses lower-cased ietf codes (e.g. en-us) """ assert format == "srt", "We only support srt download at the moment." # srt map deals with amara, so uses ietf codes (e.g. en-us) api_info_map = softload_json(SRTS_JSON_FILEPATH, raises=True) # get amara id amara_code = api_info_map.get(youtube_id, {}).get("amara_code") # make request # Please see http://amara.readthedocs.org/en/latest/api.html base_url = "https://amara.org/api2/partners/videos" resp = make_request(AMARA_HEADERS, "%s/%s/languages/%s/subtitles/?format=srt" % ( base_url, amara_code, lang_code.lower(), )) if isinstance(resp, basestring): return resp else: # return the subtitle text, replacing empty subtitle lines with # spaces to make the FLV player happy try: resp.encoding = "UTF-8" response = (resp.text or u"") \ .replace("\n\n\n", "\n \n\n") \ .replace("\r\n\r\n\r\n", "\r\n \r\n\r\n") except Exception as e: logging.error(e) response = "client-error" return response
def execute(self, iterations=1): if iterations < 1: iterations = 1 if hasattr(self, 'max_iterations'): if iterations > self.max_iterations: iterations = self.max_iterations self.return_dict['iterations'] = iterations self.return_dict['individual_elapsed'] = {} self.return_dict['post_execute_info'] = {} self.return_dict['exceptions'] = {} for i in range(iterations): self.return_dict['exceptions'][i+1] = [] start_time = time.time() try: self._execute() self.return_dict['individual_elapsed'][i+1] = time.time() - start_time except Exception as e: self.return_dict['individual_elapsed'][i+1] = None self.return_dict['exceptions'][i+1].append(e) logging.error("Exception running execute: %s" % e) try: self.return_dict['post_execute_info'][i+1] = self._get_post_execute_info() except Exception as e: self.return_dict['post_execute_info'][i+1] = None self.return_dict['exceptions'][i+1].append(e) logging.error("Exception getting execute info: %s" % e) mean = lambda vals: sum(vals)/float(len(vals)) if len(vals) else None self.return_dict['average_elapsed'] = mean([v for v in self.return_dict['individual_elapsed'].values() if v is not None]) try: self._teardown() except Exception as e: logging.error(e) return self.return_dict
def download_srt_from_3rd_party(lang_codes=None, **kwargs): """Download subtitles specified by command line args""" lang_codes = lang_codes or get_all_prepped_lang_codes() bad_languages = {} for lang_code in lang_codes: lang_code = lcode_to_ietf(lang_code) lang_code = get_supported_language_map(lang_code)['amara'] try: lang_map_filepath = get_lang_map_filepath(lang_code) if not os.path.exists(lang_map_filepath): videos = {} # happens if an unknown set for subtitles. else: with open(lang_map_filepath, "r") as fp: videos = json.load(fp) except Exception as e: error_msg = "Error in subtitles metadata file for %s: %s" % (lang_code, e) logging.error(error_msg) bad_languages[lang_code] = error_msg continue try: download_if_criteria_met(videos, lang_code=lang_code, **kwargs) except Exception as e: error_msg = "Error downloading subtitles for %s: %s" % (lang_code, e) logging.error(error_msg) bad_languages[lang_code] = error_msg continue # now report final results if bad_languages: outstr = "Failed to download subtitles for the following languages: %s" % (bad_languages.keys()) outstr += "\n" + str(bad_languages) logging.error(outstr)
def update_all_central_callback(request): """ Callback after authentication. Parses out the request token verification. Then finishes the request by getting an auth token. """ if not "ACCESS_TOKEN" in request.session: finish_auth(request) exercises = get_api_resource(request, "/api/v1/user/exercises") videos = get_api_resource(request, "/api/v1/user/videos") node_cache = get_node_cache() # Collate videos video_logs = [] for video in videos: # Assume that KA videos are all english-language, not dubbed (for now) video_id = youtube_id = video.get('video', {}).get('youtube_id', "") # Only save videos with progress if not video.get('seconds_watched', None): continue # Only save video logs for videos that we recognize. if video_id not in node_cache["Video"]: logging.warn("Skipping unknown video %s" % video_id) continue try: video_logs.append({ "video_id": video_id, "youtube_id": youtube_id, "total_seconds_watched": video['seconds_watched'], "points": VideoLog.calc_points(video['seconds_watched'], video['duration']), "complete": video['completed'], "completion_timestamp": convert_ka_date(video['last_watched']) if video['completed'] else None, }) logging.debug("Got video log for %s: %s" % (video_id, video_logs[-1])) except KeyError: # logging.error("Could not save video log for data with missing values: %s" % video) # Collate exercises exercise_logs = [] for exercise in exercises: # Only save exercises that have any progress. if not exercise.get('last_done', None): continue # Only save video logs for videos that we recognize. slug = exercise.get('exercise', "") if slug not in node_cache['Exercise']: logging.warn("Skipping unknown video %s" % slug) continue try: completed = exercise['streak'] >= 10 basepoints = node_cache['Exercise'][slug][0]['basepoints'] exercise_logs.append({ "exercise_id": slug, "streak_progress": min(100, 100 * exercise['streak']/10), # duplicates logic elsewhere "attempts": exercise['total_done'], "points": ExerciseLog.calc_points(basepoints, ncorrect=exercise['streak'], add_randomness=False), # no randomness when importing from KA "complete": completed, "attempts_before_completion": exercise['total_done'] if not exercise['practiced'] else None, #can't figure this out if they practiced after mastery. "completion_timestamp": convert_ka_date(exercise['proficient_date']) if completed else None, }) logging.debug("Got exercise log for %s: %s" % (slug, exercise_logs[-1])) except KeyError: logging.error("Could not save exercise log for data with missing values: %s" % exercise) # POST the data back to the distributed server try: dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None logging.debug("POST'ing to %s" % request.session["distributed_callback_url"]) response = requests.post( request.session["distributed_callback_url"], cookies={ "csrftoken": request.session["distributed_csrf_token"] }, data = { "csrfmiddlewaretoken": request.session["distributed_csrf_token"], "video_logs": json.dumps(video_logs, default=dthandler), "exercise_logs": json.dumps(exercise_logs, default=dthandler), "user_id": request.session["distributed_user_id"], } ) logging.debug("Response (%d): %s" % (response.status_code, response.content)) except requests.exceptions.ConnectionError as e: return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], { "message_type": "error", "message": _("Could not connect to your KA Lite installation to share Khan Academy data."), "message_id": "id_khanload", })) except Exception as e: return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], { "message_type": "error", "message": _("Failure to send data to your KA Lite installation: %s") % e, "message_id": "id_khanload", })) try: json_response = json.loads(response.content) if not isinstance(json_response, dict) or len(json_response) != 1: # Could not validate the message is a single key-value pair raise Exception(_("Unexpected response format from your KA Lite installation.")) message_type = json_response.keys()[0] message = json_response.values()[0] except ValueError as e: message_type = "error" message = unicode(e) except Exception as e: message_type = "error" message = _("Loading json object: %s") % e # If something broke on the distribute d server, we are SCREWED. # For now, just show the error to users. # # Ultimately, we have a message, would like to share with the distributed server. # if response.status_code != 200: # return HttpResponseServerError(response.content) return HttpResponseRedirect(set_query_params(request.session["distributed_redirect_url"], { "message_type": message_type, "message": message, "message_id": "id_khanload", }))
def generate_dubbed_video_mappings(download_url=None, csv_data=None): """ Function to do the heavy lifting in getting the dubbed videos map. Could be moved into utils """ if not download_url: download_url = SPREADSHEET_BASE_URL params = {'key': SPREADSHEET_ID, 'gid': SPREADSHEET_GID, 'output': SPREADSHEET_EXPORT_FORMAT} else: params = {} if not csv_data: logging.info("Downloading dubbed video data from %s" % download_url) response = requests.get(download_url, params=params) if response.status_code != 200: raise CommandError("Failed to download dubbed video CSV data: status=%s" % response.status) csv_data = response.content # This CSV file is in standard format: separated by ",", quoted by '"' logging.info("Parsing csv file.") reader = csv.reader(StringIO(csv_data)) # Build a two-level video map. # First key: language name # Second key: english youtube ID # Value: corresponding youtube ID in the new language. video_map = {} row_num = -1 try: # Loop through each row in the spreadsheet. while (True): row_num += 1 row = reader.next() if row_num < 4: # Rows 1-4 are crap. continue elif row_num == 4: # Row 5 is the header row. header_row = [v.lower() for v in row] # lcase all header row values (including language names) slug_idx = header_row.index("titled id") english_idx = header_row.index("english") assert slug_idx != -1, "Video slug column header should be found." assert english_idx != -1, "English video column header should be found." else: # Rows 6 and beyond are data. assert len(row) == len(header_row), "Values line length equals headers line length" # Grab the slug and english video ID. video_slug = row[slug_idx] english_video_id = row[english_idx] assert english_video_id, "English Video ID should not be empty" assert video_slug, "Slug should not be empty" # English video is the first video ID column, # and following columns (until the end) are other languages. # Loop through those columns and, if a video exists, # add it to the dictionary. for idx in range(english_idx, len(row)): if not row[idx]: # make sure there's a dubbed video continue lang = header_row[idx] if lang not in video_map: # add the first level if it doesn't exist video_map[lang] = {} dubbed_youtube_id = row[idx] if english_video_id == dubbed_youtube_id and lang != "english": logging.error("Removing entry for (%s, %s): dubbed and english youtube ID are the same." % (lang, english_video_id)) #elif dubbed_youtube_id in video_map[lang].values(): # Talked to Bilal, and this is actually supposed to be OK. Would throw us for a loop! # For now, just keep one. #for key in video_map[lang].keys(): # if video_map[lang][key] == dubbed_youtube_id: # del video_map[lang][key] # break #logging.error("Removing entry for (%s, %s): the same dubbed video ID is used in two places, and we can only keep one in our current system." % (lang, english_video_id)) else: video_map[lang][english_video_id] = row[idx] # add the corresponding video id for the video, in this language. except StopIteration: # The loop ends when the CSV file hits the end and throws a StopIteration pass # Now, validate the mappings with our topic data known_videos = get_node_cache("Video").keys() missing_videos = set(known_videos) - set(video_map["english"].keys()) extra_videos = set(video_map["english"].keys()) - set(known_videos) if missing_videos: logging.warn("There are %d known videos not in the list of dubbed videos" % len(missing_videos)) logging.warn("Adding missing English videos to English dubbed video map") for video in missing_videos: video_map["english"][video] = video if extra_videos: logging.warn("There are %d videos in the list of dubbed videos that we have never heard of." % len(extra_videos)) return (video_map, csv_data)
def login(request, facility): facility_id = facility and facility.id or None facilities = list(Facility.objects.all()) # Fix for #1211: refresh cached facility info when it's free and relevant refresh_session_facility_info(request, facility_count=len(facilities)) if request.method == 'POST': # log out any Django user or facility user logout(request) username = request.POST.get("username", "") password = request.POST.get("password", "") # first try logging in as a Django user user = authenticate(username=username, password=password) if user: auth_login(request, user) return HttpResponseRedirect(request.next or reverse("easy_admin")) # try logging in as a facility user form = LoginForm(data=request.POST, request=request, initial={"facility": facility_id}) if form.is_valid(): user = form.get_user() try: UserLog.begin_user_activity(user, activity_type="login", language=request.language) # Success! Log the event (ignoring validation failures) except ValidationError as e: logging.error("Failed to begin_user_activity upon login: %s" % e) request.session["facility_user"] = user messages.success(request, _("You've been logged in! We hope you enjoy your time with KA Lite ") + _("-- be sure to log out when you finish.")) # Send them back from whence they came landing_page = form.cleaned_data["callback_url"] if not landing_page: # Just going back to the homepage? We can do better than that. landing_page = reverse("coach_reports") if form.get_user().is_teacher else None landing_page = landing_page or (reverse("account_management") if not package_selected("RPi") else reverse("homepage")) return HttpResponseRedirect(form.non_field_errors() or request.next or landing_page) else: messages.error( request, _("There was an error logging you in. Please correct any errors listed below, and try again."), ) else: # render the unbound login form referer = urlparse.urlparse(request.META["HTTP_REFERER"]).path if request.META.get("HTTP_REFERER") else None # never use the homepage as the referer if referer in [reverse("homepage"), reverse("add_facility_student")]: referer = None form = LoginForm(initial={"facility": facility_id, "callback_url": referer}) return { "form": form, "facilities": facilities, "sign_up_url": reverse("add_facility_student"), }
def api_data(request, xaxis="", yaxis=""): """Request contains information about what data are requested (who, what, and how). Response should be a JSON object * data contains the data, structred by user and then datatype * the rest of the data is metadata, useful for displaying detailed info about data. """ # Get the request form form = get_data_form(request, xaxis=xaxis, yaxis=yaxis) # (data=request.REQUEST) # Query out the data: who? if form.data.get("user"): facility = [] groups = [] users = [get_object_or_404(FacilityUser, id=form.data.get("user"))] elif form.data.get("group"): facility = [] groups = [get_object_or_404(FacilityGroup, id=form.data.get("group"))] users = FacilityUser.objects.filter(group=form.data.get("group"), is_teacher=False).order_by("last_name", "first_name") elif form.data.get("facility"): facility = get_object_or_404(Facility, id=form.data.get("facility")) groups = FacilityGroup.objects.filter(facility__in=[form.data.get("facility")]) users = FacilityUser.objects.filter(facility__in=[form.data.get("facility")], is_teacher=False).order_by("last_name", "first_name") else: return HttpResponseNotFound(_("Did not specify facility, group, nor user.")) # Query out the data: where? if not form.data.get("topic_path"): return HttpResponseNotFound(_("Must specify a topic path")) # Query out the data: what? computed_data = compute_data(data_types=[form.data.get("xaxis"), form.data.get("yaxis")], who=users, where=form.data.get("topic_path")) json_data = { "data": computed_data["data"], "exercises": computed_data["exercises"], "videos": computed_data["videos"], "users": dict(zip([u.id for u in users], ["%s, %s" % (u.last_name, u.first_name) for u in users] )), "groups": dict(zip([g.id for g in groups], dict(zip(["id", "name"], [(g.id, g.name) for g in groups])), )), "facility": None if not facility else { "name": facility.name, "id": facility.id, } } if "facility_user" in request.session: try: # Log a "begin" and end here user = request.session["facility_user"] UserLog.begin_user_activity(user, activity_type="coachreport") UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers UserLog.end_user_activity(user, activity_type="coachreport") except ValidationError as e: # Never report this error; don't want this logging to block other functionality. logging.error("Failed to update Teacher userlog activity login: %s" % e) # Now we have data, stream it back with a handler for date-times return JsonResponse(json_data)
def student_view_context(request, xaxis="pct_mastery", yaxis="ex:attempts"): """ Context done separately, to be importable for similar pages. """ user = get_user_from_request(request=request) if not user: raise Http404("User not found.") node_cache = get_node_cache() topic_ids = get_knowledgemap_topics() topic_ids += [ch["id"] for node in get_topic_tree()["children"] for ch in node["children"] if node["id"] != "math"] topics = [node_cache["Topic"][id][0] for id in topic_ids] user_id = user.id exercise_logs = list(ExerciseLog.objects \ .filter(user=user) \ .values("exercise_id", "complete", "points", "attempts", "streak_progress", "struggling", "completion_timestamp")) video_logs = list(VideoLog.objects \ .filter(user=user) \ .values("video_id", "complete", "total_seconds_watched", "points", "completion_timestamp")) exercise_sparklines = dict() stats = dict() topic_exercises = dict() topic_videos = dict() exercises_by_topic = dict() videos_by_topic = dict() # Categorize every exercise log into a "midlevel" exercise for elog in exercise_logs: if not elog["exercise_id"] in node_cache["Exercise"]: # Sometimes KA updates their topic tree and eliminates exercises; # we also want to support 3rd party switching of trees arbitrarily. logging.debug("Skip unknown exercise log for %s/%s" % (user_id, elog["exercise_id"])) continue parent_ids = [topic for ex in node_cache["Exercise"][elog["exercise_id"]] for topic in ex["ancestor_ids"]] topic = set(parent_ids).intersection(set(topic_ids)) if not topic: logging.error("Could not find a topic for exercise %s (parents=%s)" % (elog["exercise_id"], parent_ids)) continue topic = topic.pop() if not topic in topic_exercises: topic_exercises[topic] = get_topic_exercises(path=node_cache["Topic"][topic][0]["path"]) exercises_by_topic[topic] = exercises_by_topic.get(topic, []) + [elog] # Categorize every video log into a "midlevel" exercise. for vlog in video_logs: if not vlog["video_id"] in node_cache["Video"]: # Sometimes KA updates their topic tree and eliminates videos; # we also want to support 3rd party switching of trees arbitrarily. logging.debug("Skip unknown video log for %s/%s" % (user_id, vlog["video_id"])) continue parent_ids = [topic for vid in node_cache["Video"][vlog["video_id"]] for topic in vid["ancestor_ids"]] topic = set(parent_ids).intersection(set(topic_ids)) if not topic: logging.error("Could not find a topic for video %s (parents=%s)" % (vlog["video_id"], parent_ids)) continue topic = topic.pop() if not topic in topic_videos: topic_videos[topic] = get_topic_videos(path=node_cache["Topic"][topic][0]["path"]) videos_by_topic[topic] = videos_by_topic.get(topic, []) + [vlog] # Now compute stats for id in topic_ids:#set(topic_exercises.keys()).union(set(topic_videos.keys())): n_exercises = len(topic_exercises.get(id, [])) n_videos = len(topic_videos.get(id, [])) exercises = exercises_by_topic.get(id, []) videos = videos_by_topic.get(id, []) n_exercises_touched = len(exercises) n_videos_touched = len(videos) exercise_sparklines[id] = [el["completion_timestamp"] for el in filter(lambda n: n["complete"], exercises)] # total streak currently a pct, but expressed in max 100; convert to # proportion (like other percentages here) stats[id] = { "ex:pct_mastery": 0 if not n_exercises_touched else sum([el["complete"] for el in exercises]) / float(n_exercises), "ex:pct_started": 0 if not n_exercises_touched else n_exercises_touched / float(n_exercises), "ex:average_points": 0 if not n_exercises_touched else sum([el["points"] for el in exercises]) / float(n_exercises_touched), "ex:average_attempts": 0 if not n_exercises_touched else sum([el["attempts"] for el in exercises]) / float(n_exercises_touched), "ex:average_streak": 0 if not n_exercises_touched else sum([el["streak_progress"] for el in exercises]) / float(n_exercises_touched) / 100., "ex:total_struggling": 0 if not n_exercises_touched else sum([el["struggling"] for el in exercises]), "ex:last_completed": None if not n_exercises_touched else max_none([el["completion_timestamp"] or None for el in exercises]), "vid:pct_started": 0 if not n_videos_touched else n_videos_touched / float(n_videos), "vid:pct_completed": 0 if not n_videos_touched else sum([vl["complete"] for vl in videos]) / float(n_videos), "vid:total_minutes": 0 if not n_videos_touched else sum([vl["total_seconds_watched"] for vl in videos]) / 60., "vid:average_points": 0. if not n_videos_touched else float(sum([vl["points"] for vl in videos]) / float(n_videos_touched)), "vid:last_completed": None if not n_videos_touched else max_none([vl["completion_timestamp"] or None for vl in videos]), } context = plotting_metadata_context(request) return { "form": context["form"], "groups": context["groups"], "facilities": context["facilities"], "student": user, "topics": topics, "exercises": topic_exercises, "exercise_logs": exercises_by_topic, "video_logs": videos_by_topic, "exercise_sparklines": exercise_sparklines, "no_data": not exercise_logs and not video_logs, "stats": stats, "stat_defs": [ # this order determines the order of display {"key": "ex:pct_mastery", "title": _("% Mastery"), "type": "pct"}, {"key": "ex:pct_started", "title": _("% Started"), "type": "pct"}, {"key": "ex:average_points", "title": _("Average Points"), "type": "float"}, {"key": "ex:average_attempts", "title": _("Average Attempts"), "type": "float"}, {"key": "ex:average_streak", "title": _("Average Streak"), "type": "pct"}, {"key": "ex:total_struggling", "title": _("Struggling"), "type": "int"}, {"key": "ex:last_completed", "title": _("Last Completed"), "type": "date"}, {"key": "vid:pct_completed", "title": _("% Completed"), "type": "pct"}, {"key": "vid:pct_started", "title": _("% Started"), "type": "pct"}, {"key": "vid:total_minutes", "title": _("Average Minutes Watched"),"type": "float"}, {"key": "vid:average_points", "title": _("Average Points"), "type": "float"}, {"key": "vid:last_completed", "title": _("Last Completed"), "type": "date"}, ] }
def tabular_view(request, facility, report_type="exercise"): """Tabular view also gets data server-side.""" # Define how students are ordered--used to be as efficient as possible. student_ordering = ["last_name", "first_name", "username"] # Get a list of topics (sorted) and groups topics = [get_node_cache("Topic").get(tid) for tid in get_knowledgemap_topics()] (groups, facilities) = get_accessible_objects_from_logged_in_user(request, facility=facility) context = plotting_metadata_context(request, facility=facility) context.update({ # For translators: the following two translations are nouns "report_types": (_("exercise"), _("video")), "request_report_type": report_type, "topics": [{"id": t[0]["id"], "title": t[0]["title"]} for t in topics if t], }) # get querystring info topic_id = request.GET.get("topic", "") # No valid data; just show generic if not topic_id or not re.match("^[\w\-]+$", topic_id): return context group_id = request.GET.get("group", "") if group_id: # Narrow by group users = FacilityUser.objects.filter( group=group_id, is_teacher=False).order_by(*student_ordering) elif facility: # Narrow by facility search_groups = [groups_dict["groups"] for groups_dict in groups if groups_dict["facility"] == facility.id] assert len(search_groups) <= 1, "Should only have one or zero matches." # Return groups and ungrouped search_groups = search_groups[0] # make sure to include ungrouped students users = FacilityUser.objects.filter( Q(group__in=search_groups) | Q(group=None, facility=facility), is_teacher=False).order_by(*student_ordering) else: # Show all (including ungrouped) for groups_dict in groups: search_groups += groups_dict["groups"] users = FacilityUser.objects.filter( Q(group__in=search_groups) | Q(group=None), is_teacher=False).order_by(*student_ordering) # We have enough data to render over a group of students # Get type-specific information if report_type == "exercise": # Fill in exercises exercises = get_topic_exercises(topic_id=topic_id) exercises = sorted(exercises, key=lambda e: (e["h_position"], e["v_position"])) context["exercises"] = exercises # More code, but much faster exercise_names = [ex["name"] for ex in context["exercises"]] # Get students context["students"] = [] exlogs = ExerciseLog.objects \ .filter(user__in=users, exercise_id__in=exercise_names) \ .order_by(*["user__%s" % field for field in student_ordering]) \ .values("user__id", "struggling", "complete", "exercise_id") exlogs = list(exlogs) # force the query to be evaluated exlog_idx = 0 for user in users: log_table = {} while exlog_idx < len(exlogs) and exlogs[exlog_idx]["user__id"] == user.id: log_table[exlogs[exlog_idx]["exercise_id"]] = exlogs[exlog_idx] exlog_idx += 1 context["students"].append({ # this could be DRYer "first_name": user.first_name, "last_name": user.last_name, "username": user.username, "name": user.get_name(), "id": user.id, "exercise_logs": log_table, }) elif report_type == "video": # Fill in videos context["videos"] = get_topic_videos(topic_id=topic_id) # More code, but much faster video_ids = [vid["id"] for vid in context["videos"]] # Get students context["students"] = [] vidlogs = VideoLog.objects \ .filter(user__in=users, video_id__in=video_ids) \ .order_by(*["user__%s" % field for field in student_ordering])\ .values("user__id", "complete", "video_id", "total_seconds_watched", "points") vidlogs = list(vidlogs) # force the query to be executed now vidlog_idx = 0 for user in users: log_table = {} while vidlog_idx < len(vidlogs) and vidlogs[vidlog_idx]["user__id"] == user.id: log_table[vidlogs[vidlog_idx]["video_id"]] = vidlogs[vidlog_idx] vidlog_idx += 1 context["students"].append({ # this could be DRYer "first_name": user.first_name, "last_name": user.last_name, "username": user.username, "name": user.get_name(), "id": user.id, "video_logs": log_table, }) else: raise Http404(_("Unknown report_type: %(report_type)s") % {"report_type": report_type}) if "facility_user" in request.session: try: # Log a "begin" and end here user = request.session["facility_user"] UserLog.begin_user_activity(user, activity_type="coachreport") UserLog.update_user_activity(user, activity_type="login") # to track active login time for teachers UserLog.end_user_activity(user, activity_type="coachreport") except ValidationError as e: # Never report this error; don't want this logging to block other functionality. logging.error("Failed to update Teacher userlog activity login: %s" % e) return context