def handle(self, *args, **options): """Handler for command.""" task_number = options['task_number'] if len(args) == 2: course_id = SlashSeparatedCourseKey.from_deprecated_string(args[0]) usage_key = course_id.make_usage_key_from_deprecated_string(args[1]) else: print self.help return try: course = get_course(course_id) except ValueError as err: print err return descriptor = modulestore().get_item(usage_key, depth=0) if descriptor is None: print "Location {0} not found in course".format(usage_key) return try: enrolled_students = CourseEnrollment.users_enrolled_in(course_id) print "Total students enrolled in {0}: {1}".format(course_id, enrolled_students.count()) calculate_task_statistics(enrolled_students, course, usage_key, task_number) except KeyboardInterrupt: print "\nOperation Cancelled"
def handle(self, *args, **options): """Handler for command.""" task_number = options['task_number'] if len(args) == 2: course_id = args[0] location = args[1] else: print self.help return try: course = get_course(course_id) except ValueError as err: print err return descriptor = modulestore().get_instance(course.id, location, depth=0) if descriptor is None: print "Location {0} not found in course".format(location) return try: enrolled_students = CourseEnrollment.users_enrolled_in(course_id) print "Total students enrolled in {0}: {1}".format(course_id, enrolled_students.count()) calculate_task_statistics(enrolled_students, course, location, task_number) except KeyboardInterrupt: print "\nOperation Cancelled"
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) task_progress = TaskProgress(action_name, enrolled_students.count(), start_time) fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}' task_info_string = fmt.format( task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None, entry_id=_entry_id, course_id=course_id, task_input=_task_input ) TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name) course = get_course_by_id(course_id) course_is_cohorted = is_course_cohorted(course.id) cohorts_header = ['Cohort Name'] if course_is_cohorted else [] experiment_partitions = get_split_user_partitions(course.user_partitions) group_configs_header = [u'Experiment Group ({})'.format(partition.name) for partition in experiment_partitions] # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] current_step = {'step': 'Calculating Grades'} total_enrolled_students = enrolled_students.count() student_counter = 0 TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s', task_info_string, action_name, current_step, total_enrolled_students ) for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students): # Periodically update task status (this is a cache write) if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) task_progress.attempted += 1 # Now add a log entry after certain intervals to get a hint that task is in progress student_counter += 1 if student_counter % 1000 == 0: TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation in-progress for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students ) if gradeset: # We were able to successfully grade this student for this course. task_progress.succeeded += 1 if not header: header = [section['label'] for section in gradeset[u'section_breakdown']] rows.append( ["id", "email", "username", "grade"] + header + cohorts_header + group_configs_header ) percents = { section['label']: section.get('percent', 0.0) for section in gradeset[u'section_breakdown'] if 'label' in section } cohorts_group_name = [] if course_is_cohorted: group = get_cohort(student, course_id, assign=False) cohorts_group_name.append(group.name if group else '') group_configs_group_names = [] for partition in experiment_partitions: group = LmsPartitionService(student, course_id).get_group(partition, assign=False) group_configs_group_names.append(group.name if group else '') # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append( [student.id, student.email, student.username, gradeset['percent']] + row_percents + cohorts_group_name + group_configs_group_names ) else: # An empty gradeset means we failed to grade a student. task_progress.failed += 1 err_rows.append([student.id, student.username, err_msg]) TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation completed for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students ) # By this point, we've got the rows we're going to stuff into our CSV files. current_step = {'step': 'Uploading CSVs'} task_progress.update_task_state(extra_meta=current_step) TASK_LOG.info(u'%s, Task type: %s, Current step: %s', task_info_string, action_name, current_step) # Perform the actual upload upload_csv_to_report_store(rows, 'grade_report', course_id, start_date) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: upload_csv_to_report_store(err_rows, 'grade_report_err', course_id, start_date) # One last update before we close out... TASK_LOG.info(u'%s, Task type: %s, Finalizing grade task', task_info_string, action_name) return task_progress.update_task_state(extra_meta=current_step)
def push_grades_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `GradesStore`. Once created, the files can be accessed by instantiating another `GradesStore` (via `GradesStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in GradesStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) num_total = enrolled_students.count() num_attempted = 0 num_succeeded = 0 num_failed = 0 curr_step = "Calculating Grades" def update_task_progress(): """Return a dict containing info about current task""" current_time = datetime.now(UTC) progress = { 'action_name': action_name, 'attempted': num_attempted, 'succeeded': num_succeeded, 'failed': num_failed, 'total': num_total, 'duration_ms': int((current_time - start_time).total_seconds() * 1000), 'step': curr_step, } _get_current_task().update_state(state=PROGRESS, meta=progress) return progress # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students): # Periodically update task status (this is a cache write) if num_attempted % status_interval == 0: update_task_progress() num_attempted += 1 if gradeset: # We were able to successfully grade this student for this course. num_succeeded += 1 if not header: # Encode the header row in utf-8 encoding in case there are unicode characters header = [section['label'].encode('utf-8') for section in gradeset[u'section_breakdown']] rows.append(["id", "email", "username", "grade"] + header) percents = { section['label']: section.get('percent', 0.0) for section in gradeset[u'section_breakdown'] if 'label' in section } # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append([student.id, student.email, student.username, gradeset['percent']] + row_percents) else: # An empty gradeset means we failed to grade a student. num_failed += 1 err_rows.append([student.id, student.username, err_msg]) # By this point, we've got the rows we're going to stuff into our CSV files. curr_step = "Uploading CSVs" update_task_progress() # Generate parts of the file name timestamp_str = start_time.strftime("%Y-%m-%d-%H%M") course_id_prefix = urllib.quote(course_id.replace("/", "_")) # Perform the actual upload grades_store = GradesStore.from_config() grades_store.store_rows( course_id, "{}_grade_report_{}.csv".format(course_id_prefix, timestamp_str), rows ) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: grades_store.store_rows( course_id, "{}_grade_report_{}_err.csv".format(course_id_prefix, timestamp_str), err_rows ) # One last update before we close out... return update_task_progress()
def index(request, course_id): # Request data course_key = get_course_key(course_id) course = get_course_module(course_key) user = request.user staff_access = has_access(request.user, 'staff', course) instructor_access = has_access(request.user, 'instructor', course) masq = setup_masquerade(request, staff_access) # allow staff to toggle masquerade on info page studio_url = get_studio_url(course, 'course_info') reverifications = fetch_reverify_banner_info(request, course_key) #course = get_course_with_access(request.user, action='load', course_key=course_key, depth=None, check_if_enrolled=False) # Proficiency and pass limit pass_limit = get_course_grade_cutoff(course) proficiency_limit = (1 - pass_limit) / 2 + pass_limit usernames_in = [] for student in CourseEnrollment.users_enrolled_in(course_key): usernames_in.append(student.username.encode('utf-8')) # Data for visualization in JSON user_for_charts = '#average' if (staff_access or instructor_access) else user kwargs = { 'qualifiers': {'category': 'video', }, } # This returns video descriptors in the order they appear on the course video_descriptors = videos_problems_in(course)[0] video_durations = get_info_videos_descriptors(video_descriptors)[2] video_ids_str = [] course_video_names = [] for descriptor in video_descriptors: video_ids_str.append((course_key.make_usage_key('video', descriptor.location.name))._to_string()) course_video_names.append(descriptor.display_name_with_default) if len(video_descriptors) > 0: first_video_id = course_key.make_usage_key('video', video_descriptors[0].location.name) # Video progress visualization. Video percentage seen total and non-overlapped. video_names, avg_video_time, video_percentages = get_module_consumption(user_for_charts, course_key, 'video', 'video_progress') if avg_video_time != []: all_video_time_percent = map(truediv, avg_video_time, video_durations) all_video_time_percent = [int(round(x*100,0)) for x in all_video_time_percent] else: all_video_time_percent = avg_video_time column_headers = ['Video', 'Different video time', 'Total video time'] video_prog_json = ready_for_arraytodatatable(column_headers, video_names, video_percentages, all_video_time_percent) video_names, all_video_time = get_module_consumption(user_for_charts, course_key, 'video', 'total_time_vid_prob')[0:2] # Time spent on every video resource column_headers = ['Video', 'Time watched'] video_distrib_json = ready_for_arraytodatatable(column_headers, video_names, all_video_time) # Video events dispersion within video length scatter_array = get_video_events_info(user_for_charts, first_video_id) # Repetitions per video intervals user_for_vid_intervals = '#class_total_times' if user_for_charts == '#average' else user_for_charts video_intervals_array = get_user_video_intervals(user_for_vid_intervals, first_video_id) # Case no videos in course else: video_names = None video_prog_json = simplejson.dumps(None) video_distrib_json = simplejson.dumps(None) scatter_array = simplejson.dumps(None) video_intervals_array = simplejson.dumps(None) # Time spent on every problem resource problem_names, time_x_problem = get_module_consumption(user_for_charts, course_key, 'problem', 'total_time_vid_prob')[0:2] column_headers = ['Problem', 'Time on problem'] problem_distrib_json = ready_for_arraytodatatable(column_headers, problem_names, time_x_problem) # Daily time spent on video and/or problem resources video_days, video_daily_time = get_daily_consumption(user_for_charts, course_key, 'video') problem_days, problem_daily_time = get_daily_consumption(user_for_charts, course_key, 'problem') vid_and_prob_daily_time = join_video_problem_time(video_days, video_daily_time, problem_days, problem_daily_time) #Analytics visualizations if staff_access or instructor_access: # Instructor access std_sort = get_DB_sort_course_homework(course_key) # Chapter time cs, st = get_DB_course_spent_time(course_key, student_id=ALL_STUDENTS) students_spent_time = chapter_time_to_js(cs, st) students_grades = get_DB_student_grades(course_key, student_id=ALL_STUDENTS) cs, sa = course_accesses = get_DB_course_section_accesses(course_key, student_id=ALL_STUDENTS) students_course_accesses = course_accesses_to_js(cs, sa) students_prob_vid_progress = get_DB_course_video_problem_progress(course_key, student_id=ALL_STUDENTS) students_time_schedule = get_DB_time_schedule(course_key, student_id=ALL_STUDENTS) else: # Sort homework # Chapter time std_sort = None cs, st = get_DB_course_spent_time(course_key, user.id) students_spent_time = chapter_time_to_js(cs, st) students_grades = get_DB_student_grades(course_key, user.id) cs, sa = course_accesses = get_DB_course_section_accesses(course_key, user.id) students_course_accesses = course_accesses_to_js(cs, sa) students_time_schedule = get_DB_time_schedule(course_key, user.id) students_prob_vid_progress = get_DB_course_video_problem_progress(course_key, user.id) context = {'course': course, 'request': request, 'user': user, 'user_id': user.id, 'staff_access': staff_access, 'instructor_access': instructor_access, 'masquerade': masq, 'studio_url': studio_url, 'reverifications': reverifications, 'course_id': course_id, 'students': students_to_js(get_course_students(course_key)), 'visualizations_id': VISUALIZATIONS_ID, 'std_grades_dump': dumps(students_grades), 'sort_std_dump': dumps(std_sort), 'time_dump': dumps(students_spent_time), 'accesses_dump': dumps(students_course_accesses), 'std_time_schedule_dumb': dumps(students_time_schedule), 'vid_prob_prog_dump': dumps(students_prob_vid_progress), 'pass_limit': pass_limit, 'prof_limit': proficiency_limit, 'usernames_in' : usernames_in, 'video_names' : course_video_names, 'video_ids' : video_ids_str, 'video_prog_json' : video_prog_json, 'video_distrib_json' : video_distrib_json, 'problem_distrib_json' : problem_distrib_json, 'video_intervals_array' : video_intervals_array, 'vid_and_prob_daily_time' : vid_and_prob_daily_time, 'scatter_array' : scatter_array, } return render_to_response('learning_analytics/learning_analytics.html', context)
def upload_problem_grade_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ Generate a CSV containing all students' problem grades within a given `course_id`. """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) task_progress = TaskProgress(action_name, enrolled_students.count(), start_time) # This struct encapsulates both the display names of each static item in the # header row as values as well as the django User field names of those items # as the keys. It is structured in this way to keep the values related. header_row = OrderedDict([('id', 'Student ID'), ('email', 'Email'), ('username', 'Username')]) try: course_structure = CourseStructure.objects.get(course_id=course_id) blocks = course_structure.ordered_blocks problems = _order_problems(blocks) except CourseStructure.DoesNotExist: return task_progress.update_task_state( extra_meta={'step': 'Generating course structure. Please refresh and try again.'} ) # Just generate the static fields for now. rows = [list(header_row.values()) + ['Final Grade'] + list(chain.from_iterable(problems.values()))] error_rows = [list(header_row.values()) + ['error_msg']] current_step = {'step': 'Calculating Grades'} for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students, keep_raw_scores=True): student_fields = [getattr(student, field_name) for field_name in header_row] task_progress.attempted += 1 if err_msg: # There was an error grading this student. error_rows.append(student_fields + [err_msg]) task_progress.failed += 1 continue final_grade = gradeset['percent'] # Only consider graded problems problem_scores = {unicode(score.module_id): score for score in gradeset['raw_scores'] if score.graded} earned_possible_values = list() for problem_id in problems: try: problem_score = problem_scores[problem_id] earned_possible_values.append([problem_score.earned, problem_score.possible]) except KeyError: # The student has not been graded on this problem. For example, # iterate_grades_for skips problems that students have never # seen in order to speed up report generation. It could also be # the case that the student does not have access to it (e.g. A/B # test or cohorted courseware). earned_possible_values.append(['N/A', 'N/A']) rows.append(student_fields + [final_grade] + list(chain.from_iterable(earned_possible_values))) task_progress.succeeded += 1 if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) # Perform the upload if any students have been successfully graded if len(rows) > 1: upload_csv_to_report_store(rows, 'problem_grade_report', course_id, start_date) # If there are any error rows, write them out as well if len(error_rows) > 1: upload_csv_to_report_store(error_rows, 'problem_grade_report_err', course_id, start_date) return task_progress.update_task_state(extra_meta={'step': 'Uploading CSV'})
def push_grades_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) num_total = enrolled_students.count() num_attempted = 0 num_succeeded = 0 num_failed = 0 curr_step = "Calculating Grades" def update_task_progress(): """Return a dict containing info about current task""" current_time = datetime.now(UTC) progress = { 'action_name': action_name, 'attempted': num_attempted, 'succeeded': num_succeeded, 'failed': num_failed, 'total': num_total, 'duration_ms': int( (current_time - start_time).total_seconds() * 1000), 'step': curr_step, } _get_current_task().update_state(state=PROGRESS, meta=progress) return progress # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] for student, gradeset, err_msg in iterate_grades_for( course_id, enrolled_students): # Periodically update task status (this is a cache write) if num_attempted % status_interval == 0: update_task_progress() num_attempted += 1 if gradeset: # We were able to successfully grade this student for this course. num_succeeded += 1 if not header: # Encode the header row in utf-8 encoding in case there are unicode characters header = [ section['label'].encode('utf-8') for section in gradeset[u'section_breakdown'] ] rows.append(["id", "email", "username", "grade"] + header) percents = { section['label']: section.get('percent', 0.0) for section in gradeset[u'section_breakdown'] if 'label' in section } # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append([ student.id, student.email.encode('utf-8'), student.username, gradeset['percent'] ] + row_percents) else: # An empty gradeset means we failed to grade a student. num_failed += 1 err_rows.append([student.id, student.username, err_msg]) # By this point, we've got the rows we're going to stuff into our CSV files. curr_step = "Uploading CSVs" update_task_progress() # Generate parts of the file name timestamp_str = start_time.strftime("%Y-%m-%d-%H%M") course_id_prefix = urllib.quote(course_id.to_deprecated_string().replace( "/", "_")) # Perform the actual upload report_store = ReportStore.from_config() report_store.store_rows( course_id, u"{}_grade_report_{}.csv".format(course_id_prefix, timestamp_str), rows) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: report_store.store_rows( course_id, u"{}_grade_report_{}_err.csv".format(course_id_prefix, timestamp_str), err_rows) # One last update before we close out... return update_task_progress()
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) task_progress = TaskProgress(action_name, enrolled_students.count(), start_time) # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] current_step = {'step': 'Calculating Grades'} for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students): # Periodically update task status (this is a cache write) if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) task_progress.attempted += 1 if gradeset: # We were able to successfully grade this student for this course. task_progress.succeeded += 1 if not header: # Encode the header row in utf-8 encoding in case there are unicode characters header = [section['label'].encode('utf-8') for section in gradeset[u'section_breakdown']] rows.append(["id", "email", "username", "grade"] + header) percents = { section['label']: section.get('percent', 0.0) for section in gradeset[u'section_breakdown'] if 'label' in section } # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append([student.id, student.email, student.username, gradeset['percent']] + row_percents) else: # An empty gradeset means we failed to grade a student. task_progress.failed += 1 err_rows.append([student.id, student.username, err_msg]) # By this point, we've got the rows we're going to stuff into our CSV files. current_step = {'step': 'Uploading CSVs'} task_progress.update_task_state(extra_meta=current_step) # Perform the actual upload upload_csv_to_report_store(rows, 'grade_report', course_id, start_date) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: upload_csv_to_report_store(err_rows, 'grade_report_err', course_id, start_date) # One last update before we close out... return task_progress.update_task_state(extra_meta=current_step)
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): # pylint: disable=too-many-statements """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) task_progress = TaskProgress(action_name, enrolled_students.count(), start_time) fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}' task_info_string = fmt.format( task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None, entry_id=_entry_id, course_id=course_id, task_input=_task_input) TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name) course = get_course_by_id(course_id) course_is_cohorted = is_course_cohorted(course.id) cohorts_header = ['Cohort Name'] if course_is_cohorted else [] experiment_partitions = get_split_user_partitions(course.user_partitions) group_configs_header = [ u'Experiment Group ({})'.format(partition.name) for partition in experiment_partitions ] certificate_info_header = [ 'Certificate Eligible', 'Certificate Delivered', 'Certificate Type' ] certificate_whitelist = CertificateWhitelist.objects.filter( course_id=course_id, whitelist=True) whitelisted_user_ids = [entry.user_id for entry in certificate_whitelist] # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] current_step = {'step': 'Calculating Grades'} total_enrolled_students = enrolled_students.count() student_counter = 0 TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s', task_info_string, action_name, current_step, total_enrolled_students) for student, gradeset, err_msg in iterate_grades_for( course_id, enrolled_students): # Periodically update task status (this is a cache write) if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) task_progress.attempted += 1 # Now add a log entry after certain intervals to get a hint that task is in progress student_counter += 1 if student_counter % 1000 == 0: TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation in-progress for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students) if gradeset: # We were able to successfully grade this student for this course. task_progress.succeeded += 1 if not header: header = [ section['label'] for section in gradeset[u'section_breakdown'] ] rows.append(["id", "email", "username", "grade"] + header + cohorts_header + group_configs_header + ['Enrollment Track', 'Verification Status'] + certificate_info_header) percents = { section['label']: section.get('percent', 0.0) for section in gradeset[u'section_breakdown'] if 'label' in section } cohorts_group_name = [] if course_is_cohorted: group = get_cohort(student, course_id, assign=False) cohorts_group_name.append(group.name if group else '') group_configs_group_names = [] for partition in experiment_partitions: group = LmsPartitionService(student, course_id).get_group(partition, assign=False) group_configs_group_names.append(group.name if group else '') enrollment_mode = CourseEnrollment.enrollment_mode_for_user( student, course_id)[0] verification_status = SoftwareSecurePhotoVerification.verification_status_for_user( student, course_id, enrollment_mode) certificate_info = certificate_info_for_user( student, course_id, gradeset['grade'], student.id in whitelisted_user_ids) # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append([ student.id, student.email, student.username, gradeset['percent'] ] + row_percents + cohorts_group_name + group_configs_group_names + [enrollment_mode] + [verification_status] + certificate_info) else: # An empty gradeset means we failed to grade a student. task_progress.failed += 1 err_rows.append([student.id, student.username, err_msg]) TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation completed for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students) # By this point, we've got the rows we're going to stuff into our CSV files. current_step = {'step': 'Uploading CSVs'} task_progress.update_task_state(extra_meta=current_step) TASK_LOG.info(u'%s, Task type: %s, Current step: %s', task_info_string, action_name, current_step) # Perform the actual upload upload_csv_to_report_store(rows, 'grade_report', course_id, start_date) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: upload_csv_to_report_store(err_rows, 'grade_report_err', course_id, start_date) # One last update before we close out... TASK_LOG.info(u'%s, Task type: %s, Finalizing grade task', task_info_string, action_name) return task_progress.update_task_state(extra_meta=current_step)
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a grades CSV file for all students that are enrolled, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. As we start to add more CSV downloads, it will probably be worthwhile to make a more general CSVDoc class instead of building out the rows like we do here. """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 enrolled_students = CourseEnrollment.users_enrolled_in(course_id) task_progress = TaskProgress(action_name, enrolled_students.count(), start_time) course = get_course_by_id(course_id) cohorts_header = ["Cohort Name"] if course.is_cohorted else [] partition_service = LmsPartitionService(user=None, course_id=course_id) partitions = partition_service.course_partitions group_configs_header = ["Group Configuration Group Name ({})".format(partition.name) for partition in partitions] # Loop over all our students and build our CSV lists in memory header = None rows = [] err_rows = [["id", "username", "error_msg"]] current_step = {"step": "Calculating Grades"} for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students): # Periodically update task status (this is a cache write) if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) task_progress.attempted += 1 if gradeset: # We were able to successfully grade this student for this course. task_progress.succeeded += 1 if not header: header = [section["label"] for section in gradeset[u"section_breakdown"]] rows.append(["id", "email", "username", "grade"] + header + cohorts_header + group_configs_header) percents = { section["label"]: section.get("percent", 0.0) for section in gradeset[u"section_breakdown"] if "label" in section } cohorts_group_name = [] if course.is_cohorted: group = get_cohort(student, course_id, assign=False) cohorts_group_name.append(group.name if group else "") group_configs_group_names = [] for partition in partitions: group = LmsPartitionService(student, course_id).get_group(partition, assign=False) group_configs_group_names.append(group.name if group else "") # Not everybody has the same gradable items. If the item is not # found in the user's gradeset, just assume it's a 0. The aggregated # grades for their sections and overall course will be calculated # without regard for the item they didn't have access to, so it's # possible for a student to have a 0.0 show up in their row but # still have 100% for the course. row_percents = [percents.get(label, 0.0) for label in header] rows.append( [student.id, student.email, student.username, gradeset["percent"]] + row_percents + cohorts_group_name + group_configs_group_names ) else: # An empty gradeset means we failed to grade a student. task_progress.failed += 1 err_rows.append([student.id, student.username, err_msg]) # By this point, we've got the rows we're going to stuff into our CSV files. current_step = {"step": "Uploading CSVs"} task_progress.update_task_state(extra_meta=current_step) # Perform the actual upload upload_csv_to_report_store(rows, "grade_report", course_id, start_date) # If there are any error rows (don't count the header), write them out as well if len(err_rows) > 1: upload_csv_to_report_store(err_rows, "grade_report_err", course_id, start_date) # One last update before we close out... return task_progress.update_task_state(extra_meta=current_step)
def index(request, course_id): # Request data course_key = get_course_key(course_id) course = get_course_module(course_key) user = request.user staff_access = has_access(request.user, 'staff', course) instructor_access = has_access(request.user, 'instructor', course) masq = setup_masquerade( request, staff_access) # allow staff to toggle masquerade on info page studio_url = get_studio_url(course, 'course_info') reverifications = fetch_reverify_banner_info(request, course_key) #course = get_course_with_access(request.user, action='load', course_key=course_key, depth=None, check_if_enrolled=False) # Proficiency and pass limit pass_limit = get_course_grade_cutoff(course) proficiency_limit = (1 - pass_limit) / 2 + pass_limit usernames_in = [] for student in CourseEnrollment.users_enrolled_in(course_key): usernames_in.append(student.username.encode('utf-8')) # Data for visualization in JSON user_for_charts = '#average' if (staff_access or instructor_access) else user kwargs = { 'qualifiers': { 'category': 'video', }, } # This returns video descriptors in the order they appear on the course video_descriptors = videos_problems_in(course)[0] video_durations = get_info_videos_descriptors(video_descriptors)[2] video_ids_str = [] course_video_names = [] for descriptor in video_descriptors: video_ids_str.append( (course_key.make_usage_key('video', descriptor.location.name))._to_string()) course_video_names.append(descriptor.display_name_with_default) if len(video_descriptors) > 0: first_video_id = course_key.make_usage_key( 'video', video_descriptors[0].location.name) # Video progress visualization. Video percentage seen total and non-overlapped. video_names, avg_video_time, video_percentages = get_module_consumption( user_for_charts, course_key, 'video', 'video_progress') if avg_video_time != []: all_video_time_percent = map(truediv, avg_video_time, video_durations) all_video_time_percent = [ int(round(x * 100, 0)) for x in all_video_time_percent ] else: all_video_time_percent = avg_video_time column_headers = ['Video', 'Different video time', 'Total video time'] video_prog_json = ready_for_arraytodatatable(column_headers, video_names, video_percentages, all_video_time_percent) video_names, all_video_time = get_module_consumption( user_for_charts, course_key, 'video', 'total_time_vid_prob')[0:2] # Time spent on every video resource column_headers = ['Video', 'Time watched'] video_distrib_json = ready_for_arraytodatatable( column_headers, video_names, all_video_time) # Video events dispersion within video length scatter_array = get_video_events_info(user_for_charts, first_video_id) # Repetitions per video intervals user_for_vid_intervals = '#class_total_times' if user_for_charts == '#average' else user_for_charts video_intervals_array = get_user_video_intervals( user_for_vid_intervals, first_video_id) # Case no videos in course else: video_names = None video_prog_json = simplejson.dumps(None) video_distrib_json = simplejson.dumps(None) scatter_array = simplejson.dumps(None) video_intervals_array = simplejson.dumps(None) # Time spent on every problem resource problem_names, time_x_problem = get_module_consumption( user_for_charts, course_key, 'problem', 'total_time_vid_prob')[0:2] column_headers = ['Problem', 'Time on problem'] problem_distrib_json = ready_for_arraytodatatable(column_headers, problem_names, time_x_problem) # Daily time spent on video and/or problem resources video_days, video_daily_time = get_daily_consumption( user_for_charts, course_key, 'video') problem_days, problem_daily_time = get_daily_consumption( user_for_charts, course_key, 'problem') vid_and_prob_daily_time = join_video_problem_time(video_days, video_daily_time, problem_days, problem_daily_time) #Analytics visualizations if staff_access or instructor_access: # Instructor access std_sort = get_DB_sort_course_homework(course_key) # Chapter time cs, st = get_DB_course_spent_time(course_key, student_id=ALL_STUDENTS) students_spent_time = chapter_time_to_js(cs, st) students_grades = get_DB_student_grades(course_key, student_id=ALL_STUDENTS) cs, sa = course_accesses = get_DB_course_section_accesses( course_key, student_id=ALL_STUDENTS) students_course_accesses = course_accesses_to_js(cs, sa) students_prob_vid_progress = get_DB_course_video_problem_progress( course_key, student_id=ALL_STUDENTS) students_time_schedule = get_DB_time_schedule(course_key, student_id=ALL_STUDENTS) else: # Sort homework # Chapter time std_sort = None cs, st = get_DB_course_spent_time(course_key, user.id) students_spent_time = chapter_time_to_js(cs, st) students_grades = get_DB_student_grades(course_key, user.id) cs, sa = course_accesses = get_DB_course_section_accesses( course_key, user.id) students_course_accesses = course_accesses_to_js(cs, sa) students_time_schedule = get_DB_time_schedule(course_key, user.id) students_prob_vid_progress = get_DB_course_video_problem_progress( course_key, user.id) context = { 'course': course, 'request': request, 'user': user, 'user_id': user.id, 'staff_access': staff_access, 'instructor_access': instructor_access, 'masquerade': masq, 'studio_url': studio_url, 'reverifications': reverifications, 'course_id': course_id, 'students': students_to_js(get_course_students(course_key)), 'visualizations_id': VISUALIZATIONS_ID, 'std_grades_dump': dumps(students_grades), 'sort_std_dump': dumps(std_sort), 'time_dump': dumps(students_spent_time), 'accesses_dump': dumps(students_course_accesses), 'std_time_schedule_dumb': dumps(students_time_schedule), 'vid_prob_prog_dump': dumps(students_prob_vid_progress), 'pass_limit': pass_limit, 'prof_limit': proficiency_limit, 'usernames_in': usernames_in, 'video_names': course_video_names, 'video_ids': video_ids_str, 'video_prog_json': video_prog_json, 'video_distrib_json': video_distrib_json, 'problem_distrib_json': problem_distrib_json, 'video_intervals_array': video_intervals_array, 'vid_and_prob_daily_time': vid_and_prob_daily_time, 'scatter_array': scatter_array, } return render_to_response('learning_analytics/learning_analytics.html', context)