def handle(self, *args, **options):
"""Handler for command."""
task_number = options['task_number']
if len(args) == 2:
course_id = SlashSeparatedCourseKey.from_deprecated_string(args[0])
usage_key = course_id.make_usage_key_from_deprecated_string(args[1])
else:
print self.help
return
try:
course = get_course(course_id)
except ValueError as err:
print err
return
descriptor = modulestore().get_item(usage_key, depth=0)
if descriptor is None:
print "Location {0} not found in course".format(usage_key)
return
try:
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
print "Total students enrolled in {0}: {1}".format(course_id, enrolled_students.count())
calculate_task_statistics(enrolled_students, course, usage_key, task_number)
except KeyboardInterrupt:
print "\nOperation Cancelled"
def handle(self, *args, **options):
"""Handler for command."""
task_number = options['task_number']
if len(args) == 2:
course_id = args[0]
location = args[1]
else:
print self.help
return
try:
course = get_course(course_id)
except ValueError as err:
print err
return
descriptor = modulestore().get_instance(course.id, location, depth=0)
if descriptor is None:
print "Location {0} not found in course".format(location)
return
try:
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
print "Total students enrolled in {0}: {1}".format(course_id, enrolled_students.count())
calculate_task_statistics(enrolled_students, course, location, task_number)
except KeyboardInterrupt:
print "\nOperation Cancelled"
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `ReportStore`. Once created, the files can
be accessed by instantiating another `ReportStore` (via
`ReportStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in ReportStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = time()
start_date = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
task_info_string = fmt.format(
task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
entry_id=_entry_id,
course_id=course_id,
task_input=_task_input
)
TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name)
course = get_course_by_id(course_id)
course_is_cohorted = is_course_cohorted(course.id)
cohorts_header = ['Cohort Name'] if course_is_cohorted else []
experiment_partitions = get_split_user_partitions(course.user_partitions)
group_configs_header = [u'Experiment Group ({})'.format(partition.name) for partition in experiment_partitions]
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
current_step = {'step': 'Calculating Grades'}
total_enrolled_students = enrolled_students.count()
student_counter = 0
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s',
task_info_string,
action_name,
current_step,
total_enrolled_students
)
for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if task_progress.attempted % status_interval == 0:
task_progress.update_task_state(extra_meta=current_step)
task_progress.attempted += 1
# Now add a log entry after certain intervals to get a hint that task is in progress
student_counter += 1
if student_counter % 1000 == 0:
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Grade calculation in-progress for students: %s/%s',
task_info_string,
action_name,
current_step,
student_counter,
total_enrolled_students
)
if gradeset:
# We were able to successfully grade this student for this course.
task_progress.succeeded += 1
if not header:
header = [section['label'] for section in gradeset[u'section_breakdown']]
rows.append(
["id", "email", "username", "grade"] + header + cohorts_header + group_configs_header
)
percents = {
section['label']: section.get('percent', 0.0)
for section in gradeset[u'section_breakdown']
if 'label' in section
}
cohorts_group_name = []
if course_is_cohorted:
group = get_cohort(student, course_id, assign=False)
cohorts_group_name.append(group.name if group else '')
group_configs_group_names = []
for partition in experiment_partitions:
group = LmsPartitionService(student, course_id).get_group(partition, assign=False)
group_configs_group_names.append(group.name if group else '')
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append(
[student.id, student.email, student.username, gradeset['percent']] +
row_percents + cohorts_group_name + group_configs_group_names
)
else:
# An empty gradeset means we failed to grade a student.
task_progress.failed += 1
err_rows.append([student.id, student.username, err_msg])
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Grade calculation completed for students: %s/%s',
task_info_string,
action_name,
current_step,
student_counter,
total_enrolled_students
)
# By this point, we've got the rows we're going to stuff into our CSV files.
current_step = {'step': 'Uploading CSVs'}
task_progress.update_task_state(extra_meta=current_step)
TASK_LOG.info(u'%s, Task type: %s, Current step: %s', task_info_string, action_name, current_step)
# Perform the actual upload
upload_csv_to_report_store(rows, 'grade_report', course_id, start_date)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
upload_csv_to_report_store(err_rows, 'grade_report_err', course_id, start_date)
# One last update before we close out...
TASK_LOG.info(u'%s, Task type: %s, Finalizing grade task', task_info_string, action_name)
return task_progress.update_task_state(extra_meta=current_step)
def push_grades_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `GradesStore`. Once created, the files can
be accessed by instantiating another `GradesStore` (via
`GradesStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in GradesStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
num_total = enrolled_students.count()
num_attempted = 0
num_succeeded = 0
num_failed = 0
curr_step = "Calculating Grades"
def update_task_progress():
"""Return a dict containing info about current task"""
current_time = datetime.now(UTC)
progress = {
'action_name': action_name,
'attempted': num_attempted,
'succeeded': num_succeeded,
'failed': num_failed,
'total': num_total,
'duration_ms': int((current_time - start_time).total_seconds() * 1000),
'step': curr_step,
}
_get_current_task().update_state(state=PROGRESS, meta=progress)
return progress
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if num_attempted % status_interval == 0:
update_task_progress()
num_attempted += 1
if gradeset:
# We were able to successfully grade this student for this course.
num_succeeded += 1
if not header:
# Encode the header row in utf-8 encoding in case there are unicode characters
header = [section['label'].encode('utf-8') for section in gradeset[u'section_breakdown']]
rows.append(["id", "email", "username", "grade"] + header)
percents = {
section['label']: section.get('percent', 0.0)
for section in gradeset[u'section_breakdown']
if 'label' in section
}
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append([student.id, student.email, student.username, gradeset['percent']] + row_percents)
else:
# An empty gradeset means we failed to grade a student.
num_failed += 1
err_rows.append([student.id, student.username, err_msg])
# By this point, we've got the rows we're going to stuff into our CSV files.
curr_step = "Uploading CSVs"
update_task_progress()
# Generate parts of the file name
timestamp_str = start_time.strftime("%Y-%m-%d-%H%M")
course_id_prefix = urllib.quote(course_id.replace("/", "_"))
# Perform the actual upload
grades_store = GradesStore.from_config()
grades_store.store_rows(
course_id,
"{}_grade_report_{}.csv".format(course_id_prefix, timestamp_str),
rows
)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
grades_store.store_rows(
course_id,
"{}_grade_report_{}_err.csv".format(course_id_prefix, timestamp_str),
err_rows
)
# One last update before we close out...
return update_task_progress()
def index(request, course_id):
# Request data
course_key = get_course_key(course_id)
course = get_course_module(course_key)
user = request.user
staff_access = has_access(request.user, 'staff', course)
instructor_access = has_access(request.user, 'instructor', course)
masq = setup_masquerade(request, staff_access) # allow staff to toggle masquerade on info page
studio_url = get_studio_url(course, 'course_info')
reverifications = fetch_reverify_banner_info(request, course_key)
#course = get_course_with_access(request.user, action='load', course_key=course_key, depth=None, check_if_enrolled=False)
# Proficiency and pass limit
pass_limit = get_course_grade_cutoff(course)
proficiency_limit = (1 - pass_limit) / 2 + pass_limit
usernames_in = []
for student in CourseEnrollment.users_enrolled_in(course_key):
usernames_in.append(student.username.encode('utf-8'))
# Data for visualization in JSON
user_for_charts = '#average' if (staff_access or instructor_access) else user
kwargs = {
'qualifiers': {'category': 'video', },
}
# This returns video descriptors in the order they appear on the course
video_descriptors = videos_problems_in(course)[0]
video_durations = get_info_videos_descriptors(video_descriptors)[2]
video_ids_str = []
course_video_names = []
for descriptor in video_descriptors:
video_ids_str.append((course_key.make_usage_key('video', descriptor.location.name))._to_string())
course_video_names.append(descriptor.display_name_with_default)
if len(video_descriptors) > 0:
first_video_id = course_key.make_usage_key('video', video_descriptors[0].location.name)
# Video progress visualization. Video percentage seen total and non-overlapped.
video_names, avg_video_time, video_percentages = get_module_consumption(user_for_charts, course_key, 'video', 'video_progress')
if avg_video_time != []:
all_video_time_percent = map(truediv, avg_video_time, video_durations)
all_video_time_percent = [int(round(x*100,0)) for x in all_video_time_percent]
else:
all_video_time_percent = avg_video_time
column_headers = ['Video', 'Different video time', 'Total video time']
video_prog_json = ready_for_arraytodatatable(column_headers, video_names, video_percentages, all_video_time_percent)
video_names, all_video_time = get_module_consumption(user_for_charts, course_key, 'video', 'total_time_vid_prob')[0:2]
# Time spent on every video resource
column_headers = ['Video', 'Time watched']
video_distrib_json = ready_for_arraytodatatable(column_headers, video_names, all_video_time)
# Video events dispersion within video length
scatter_array = get_video_events_info(user_for_charts, first_video_id)
# Repetitions per video intervals
user_for_vid_intervals = '#class_total_times' if user_for_charts == '#average' else user_for_charts
video_intervals_array = get_user_video_intervals(user_for_vid_intervals, first_video_id)
# Case no videos in course
else:
video_names = None
video_prog_json = simplejson.dumps(None)
video_distrib_json = simplejson.dumps(None)
scatter_array = simplejson.dumps(None)
video_intervals_array = simplejson.dumps(None)
# Time spent on every problem resource
problem_names, time_x_problem = get_module_consumption(user_for_charts, course_key, 'problem', 'total_time_vid_prob')[0:2]
column_headers = ['Problem', 'Time on problem']
problem_distrib_json = ready_for_arraytodatatable(column_headers, problem_names, time_x_problem)
# Daily time spent on video and/or problem resources
video_days, video_daily_time = get_daily_consumption(user_for_charts, course_key, 'video')
problem_days, problem_daily_time = get_daily_consumption(user_for_charts, course_key, 'problem')
vid_and_prob_daily_time = join_video_problem_time(video_days, video_daily_time, problem_days, problem_daily_time)
#Analytics visualizations
if staff_access or instructor_access:
# Instructor access
std_sort = get_DB_sort_course_homework(course_key)
# Chapter time
cs, st = get_DB_course_spent_time(course_key, student_id=ALL_STUDENTS)
students_spent_time = chapter_time_to_js(cs, st)
students_grades = get_DB_student_grades(course_key, student_id=ALL_STUDENTS)
cs, sa = course_accesses = get_DB_course_section_accesses(course_key, student_id=ALL_STUDENTS)
students_course_accesses = course_accesses_to_js(cs, sa)
students_prob_vid_progress = get_DB_course_video_problem_progress(course_key, student_id=ALL_STUDENTS)
students_time_schedule = get_DB_time_schedule(course_key, student_id=ALL_STUDENTS)
else:
# Sort homework
# Chapter time
std_sort = None
cs, st = get_DB_course_spent_time(course_key, user.id)
students_spent_time = chapter_time_to_js(cs, st)
students_grades = get_DB_student_grades(course_key, user.id)
cs, sa = course_accesses = get_DB_course_section_accesses(course_key, user.id)
students_course_accesses = course_accesses_to_js(cs, sa)
students_time_schedule = get_DB_time_schedule(course_key, user.id)
students_prob_vid_progress = get_DB_course_video_problem_progress(course_key, user.id)
context = {'course': course,
'request': request,
'user': user,
'user_id': user.id,
'staff_access': staff_access,
'instructor_access': instructor_access,
'masquerade': masq,
'studio_url': studio_url,
'reverifications': reverifications,
'course_id': course_id,
'students': students_to_js(get_course_students(course_key)),
'visualizations_id': VISUALIZATIONS_ID,
'std_grades_dump': dumps(students_grades),
'sort_std_dump': dumps(std_sort),
'time_dump': dumps(students_spent_time),
'accesses_dump': dumps(students_course_accesses),
'std_time_schedule_dumb': dumps(students_time_schedule),
'vid_prob_prog_dump': dumps(students_prob_vid_progress),
'pass_limit': pass_limit,
'prof_limit': proficiency_limit,
'usernames_in' : usernames_in,
'video_names' : course_video_names,
'video_ids' : video_ids_str,
'video_prog_json' : video_prog_json,
'video_distrib_json' : video_distrib_json,
'problem_distrib_json' : problem_distrib_json,
'video_intervals_array' : video_intervals_array,
'vid_and_prob_daily_time' : vid_and_prob_daily_time,
'scatter_array' : scatter_array, }
return render_to_response('learning_analytics/learning_analytics.html', context)
def upload_problem_grade_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
"""
Generate a CSV containing all students' problem grades within a given
`course_id`.
"""
start_time = time()
start_date = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
# This struct encapsulates both the display names of each static item in the
# header row as values as well as the django User field names of those items
# as the keys. It is structured in this way to keep the values related.
header_row = OrderedDict([('id', 'Student ID'), ('email', 'Email'), ('username', 'Username')])
try:
course_structure = CourseStructure.objects.get(course_id=course_id)
blocks = course_structure.ordered_blocks
problems = _order_problems(blocks)
except CourseStructure.DoesNotExist:
return task_progress.update_task_state(
extra_meta={'step': 'Generating course structure. Please refresh and try again.'}
)
# Just generate the static fields for now.
rows = [list(header_row.values()) + ['Final Grade'] + list(chain.from_iterable(problems.values()))]
error_rows = [list(header_row.values()) + ['error_msg']]
current_step = {'step': 'Calculating Grades'}
for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students, keep_raw_scores=True):
student_fields = [getattr(student, field_name) for field_name in header_row]
task_progress.attempted += 1
if err_msg:
# There was an error grading this student.
error_rows.append(student_fields + [err_msg])
task_progress.failed += 1
continue
final_grade = gradeset['percent']
# Only consider graded problems
problem_scores = {unicode(score.module_id): score for score in gradeset['raw_scores'] if score.graded}
earned_possible_values = list()
for problem_id in problems:
try:
problem_score = problem_scores[problem_id]
earned_possible_values.append([problem_score.earned, problem_score.possible])
except KeyError:
# The student has not been graded on this problem. For example,
# iterate_grades_for skips problems that students have never
# seen in order to speed up report generation. It could also be
# the case that the student does not have access to it (e.g. A/B
# test or cohorted courseware).
earned_possible_values.append(['N/A', 'N/A'])
rows.append(student_fields + [final_grade] + list(chain.from_iterable(earned_possible_values)))
task_progress.succeeded += 1
if task_progress.attempted % status_interval == 0:
task_progress.update_task_state(extra_meta=current_step)
# Perform the upload if any students have been successfully graded
if len(rows) > 1:
upload_csv_to_report_store(rows, 'problem_grade_report', course_id, start_date)
# If there are any error rows, write them out as well
if len(error_rows) > 1:
upload_csv_to_report_store(error_rows, 'problem_grade_report_err', course_id, start_date)
return task_progress.update_task_state(extra_meta={'step': 'Uploading CSV'})
def push_grades_to_s3(_xmodule_instance_args, _entry_id, course_id,
_task_input, action_name):
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `ReportStore`. Once created, the files can
be accessed by instantiating another `ReportStore` (via
`ReportStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in ReportStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
num_total = enrolled_students.count()
num_attempted = 0
num_succeeded = 0
num_failed = 0
curr_step = "Calculating Grades"
def update_task_progress():
"""Return a dict containing info about current task"""
current_time = datetime.now(UTC)
progress = {
'action_name': action_name,
'attempted': num_attempted,
'succeeded': num_succeeded,
'failed': num_failed,
'total': num_total,
'duration_ms': int(
(current_time - start_time).total_seconds() * 1000),
'step': curr_step,
}
_get_current_task().update_state(state=PROGRESS, meta=progress)
return progress
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
for student, gradeset, err_msg in iterate_grades_for(
course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if num_attempted % status_interval == 0:
update_task_progress()
num_attempted += 1
if gradeset:
# We were able to successfully grade this student for this course.
num_succeeded += 1
if not header:
# Encode the header row in utf-8 encoding in case there are unicode characters
header = [
section['label'].encode('utf-8')
for section in gradeset[u'section_breakdown']
]
rows.append(["id", "email", "username", "grade"] + header)
percents = {
section['label']: section.get('percent', 0.0)
for section in gradeset[u'section_breakdown']
if 'label' in section
}
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append([
student.id,
student.email.encode('utf-8'), student.username,
gradeset['percent']
] + row_percents)
else:
# An empty gradeset means we failed to grade a student.
num_failed += 1
err_rows.append([student.id, student.username, err_msg])
# By this point, we've got the rows we're going to stuff into our CSV files.
curr_step = "Uploading CSVs"
update_task_progress()
# Generate parts of the file name
timestamp_str = start_time.strftime("%Y-%m-%d-%H%M")
course_id_prefix = urllib.quote(course_id.to_deprecated_string().replace(
"/", "_"))
# Perform the actual upload
report_store = ReportStore.from_config()
report_store.store_rows(
course_id, u"{}_grade_report_{}.csv".format(course_id_prefix,
timestamp_str), rows)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
report_store.store_rows(
course_id,
u"{}_grade_report_{}_err.csv".format(course_id_prefix,
timestamp_str), err_rows)
# One last update before we close out...
return update_task_progress()
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `ReportStore`. Once created, the files can
be accessed by instantiating another `ReportStore` (via
`ReportStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in ReportStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = time()
start_date = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
current_step = {'step': 'Calculating Grades'}
for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if task_progress.attempted % status_interval == 0:
task_progress.update_task_state(extra_meta=current_step)
task_progress.attempted += 1
if gradeset:
# We were able to successfully grade this student for this course.
task_progress.succeeded += 1
if not header:
# Encode the header row in utf-8 encoding in case there are unicode characters
header = [section['label'].encode('utf-8') for section in gradeset[u'section_breakdown']]
rows.append(["id", "email", "username", "grade"] + header)
percents = {
section['label']: section.get('percent', 0.0)
for section in gradeset[u'section_breakdown']
if 'label' in section
}
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append([student.id, student.email, student.username, gradeset['percent']] + row_percents)
else:
# An empty gradeset means we failed to grade a student.
task_progress.failed += 1
err_rows.append([student.id, student.username, err_msg])
# By this point, we've got the rows we're going to stuff into our CSV files.
current_step = {'step': 'Uploading CSVs'}
task_progress.update_task_state(extra_meta=current_step)
# Perform the actual upload
upload_csv_to_report_store(rows, 'grade_report', course_id, start_date)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
upload_csv_to_report_store(err_rows, 'grade_report_err', course_id, start_date)
# One last update before we close out...
return task_progress.update_task_state(extra_meta=current_step)
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id,
_task_input, action_name): # pylint: disable=too-many-statements
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `ReportStore`. Once created, the files can
be accessed by instantiating another `ReportStore` (via
`ReportStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in ReportStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = time()
start_date = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
task_progress = TaskProgress(action_name, enrolled_students.count(),
start_time)
fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
task_info_string = fmt.format(
task_id=_xmodule_instance_args.get('task_id')
if _xmodule_instance_args is not None else None,
entry_id=_entry_id,
course_id=course_id,
task_input=_task_input)
TASK_LOG.info(u'%s, Task type: %s, Starting task execution',
task_info_string, action_name)
course = get_course_by_id(course_id)
course_is_cohorted = is_course_cohorted(course.id)
cohorts_header = ['Cohort Name'] if course_is_cohorted else []
experiment_partitions = get_split_user_partitions(course.user_partitions)
group_configs_header = [
u'Experiment Group ({})'.format(partition.name)
for partition in experiment_partitions
]
certificate_info_header = [
'Certificate Eligible', 'Certificate Delivered', 'Certificate Type'
]
certificate_whitelist = CertificateWhitelist.objects.filter(
course_id=course_id, whitelist=True)
whitelisted_user_ids = [entry.user_id for entry in certificate_whitelist]
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
current_step = {'step': 'Calculating Grades'}
total_enrolled_students = enrolled_students.count()
student_counter = 0
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s',
task_info_string, action_name, current_step, total_enrolled_students)
for student, gradeset, err_msg in iterate_grades_for(
course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if task_progress.attempted % status_interval == 0:
task_progress.update_task_state(extra_meta=current_step)
task_progress.attempted += 1
# Now add a log entry after certain intervals to get a hint that task is in progress
student_counter += 1
if student_counter % 1000 == 0:
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Grade calculation in-progress for students: %s/%s',
task_info_string, action_name, current_step, student_counter,
total_enrolled_students)
if gradeset:
# We were able to successfully grade this student for this course.
task_progress.succeeded += 1
if not header:
header = [
section['label']
for section in gradeset[u'section_breakdown']
]
rows.append(["id", "email", "username", "grade"] + header +
cohorts_header + group_configs_header +
['Enrollment Track', 'Verification Status'] +
certificate_info_header)
percents = {
section['label']: section.get('percent', 0.0)
for section in gradeset[u'section_breakdown']
if 'label' in section
}
cohorts_group_name = []
if course_is_cohorted:
group = get_cohort(student, course_id, assign=False)
cohorts_group_name.append(group.name if group else '')
group_configs_group_names = []
for partition in experiment_partitions:
group = LmsPartitionService(student,
course_id).get_group(partition,
assign=False)
group_configs_group_names.append(group.name if group else '')
enrollment_mode = CourseEnrollment.enrollment_mode_for_user(
student, course_id)[0]
verification_status = SoftwareSecurePhotoVerification.verification_status_for_user(
student, course_id, enrollment_mode)
certificate_info = certificate_info_for_user(
student, course_id, gradeset['grade'], student.id
in whitelisted_user_ids)
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append([
student.id, student.email, student.username,
gradeset['percent']
] + row_percents + cohorts_group_name + group_configs_group_names +
[enrollment_mode] + [verification_status] +
certificate_info)
else:
# An empty gradeset means we failed to grade a student.
task_progress.failed += 1
err_rows.append([student.id, student.username, err_msg])
TASK_LOG.info(
u'%s, Task type: %s, Current step: %s, Grade calculation completed for students: %s/%s',
task_info_string, action_name, current_step, student_counter,
total_enrolled_students)
# By this point, we've got the rows we're going to stuff into our CSV files.
current_step = {'step': 'Uploading CSVs'}
task_progress.update_task_state(extra_meta=current_step)
TASK_LOG.info(u'%s, Task type: %s, Current step: %s', task_info_string,
action_name, current_step)
# Perform the actual upload
upload_csv_to_report_store(rows, 'grade_report', course_id, start_date)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
upload_csv_to_report_store(err_rows, 'grade_report_err', course_id,
start_date)
# One last update before we close out...
TASK_LOG.info(u'%s, Task type: %s, Finalizing grade task',
task_info_string, action_name)
return task_progress.update_task_state(extra_meta=current_step)
def upload_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
"""
For a given `course_id`, generate a grades CSV file for all students that
are enrolled, and store using a `ReportStore`. Once created, the files can
be accessed by instantiating another `ReportStore` (via
`ReportStore.from_config()`) and calling `link_for()` on it. Writes are
buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
that are visible in ReportStore will be complete ones.
As we start to add more CSV downloads, it will probably be worthwhile to
make a more general CSVDoc class instead of building out the rows like we
do here.
"""
start_time = time()
start_date = datetime.now(UTC)
status_interval = 100
enrolled_students = CourseEnrollment.users_enrolled_in(course_id)
task_progress = TaskProgress(action_name, enrolled_students.count(), start_time)
course = get_course_by_id(course_id)
cohorts_header = ["Cohort Name"] if course.is_cohorted else []
partition_service = LmsPartitionService(user=None, course_id=course_id)
partitions = partition_service.course_partitions
group_configs_header = ["Group Configuration Group Name ({})".format(partition.name) for partition in partitions]
# Loop over all our students and build our CSV lists in memory
header = None
rows = []
err_rows = [["id", "username", "error_msg"]]
current_step = {"step": "Calculating Grades"}
for student, gradeset, err_msg in iterate_grades_for(course_id, enrolled_students):
# Periodically update task status (this is a cache write)
if task_progress.attempted % status_interval == 0:
task_progress.update_task_state(extra_meta=current_step)
task_progress.attempted += 1
if gradeset:
# We were able to successfully grade this student for this course.
task_progress.succeeded += 1
if not header:
header = [section["label"] for section in gradeset[u"section_breakdown"]]
rows.append(["id", "email", "username", "grade"] + header + cohorts_header + group_configs_header)
percents = {
section["label"]: section.get("percent", 0.0)
for section in gradeset[u"section_breakdown"]
if "label" in section
}
cohorts_group_name = []
if course.is_cohorted:
group = get_cohort(student, course_id, assign=False)
cohorts_group_name.append(group.name if group else "")
group_configs_group_names = []
for partition in partitions:
group = LmsPartitionService(student, course_id).get_group(partition, assign=False)
group_configs_group_names.append(group.name if group else "")
# Not everybody has the same gradable items. If the item is not
# found in the user's gradeset, just assume it's a 0. The aggregated
# grades for their sections and overall course will be calculated
# without regard for the item they didn't have access to, so it's
# possible for a student to have a 0.0 show up in their row but
# still have 100% for the course.
row_percents = [percents.get(label, 0.0) for label in header]
rows.append(
[student.id, student.email, student.username, gradeset["percent"]]
+ row_percents
+ cohorts_group_name
+ group_configs_group_names
)
else:
# An empty gradeset means we failed to grade a student.
task_progress.failed += 1
err_rows.append([student.id, student.username, err_msg])
# By this point, we've got the rows we're going to stuff into our CSV files.
current_step = {"step": "Uploading CSVs"}
task_progress.update_task_state(extra_meta=current_step)
# Perform the actual upload
upload_csv_to_report_store(rows, "grade_report", course_id, start_date)
# If there are any error rows (don't count the header), write them out as well
if len(err_rows) > 1:
upload_csv_to_report_store(err_rows, "grade_report_err", course_id, start_date)
# One last update before we close out...
return task_progress.update_task_state(extra_meta=current_step)
def index(request, course_id):
# Request data
course_key = get_course_key(course_id)
course = get_course_module(course_key)
user = request.user
staff_access = has_access(request.user, 'staff', course)
instructor_access = has_access(request.user, 'instructor', course)
masq = setup_masquerade(
request, staff_access) # allow staff to toggle masquerade on info page
studio_url = get_studio_url(course, 'course_info')
reverifications = fetch_reverify_banner_info(request, course_key)
#course = get_course_with_access(request.user, action='load', course_key=course_key, depth=None, check_if_enrolled=False)
# Proficiency and pass limit
pass_limit = get_course_grade_cutoff(course)
proficiency_limit = (1 - pass_limit) / 2 + pass_limit
usernames_in = []
for student in CourseEnrollment.users_enrolled_in(course_key):
usernames_in.append(student.username.encode('utf-8'))
# Data for visualization in JSON
user_for_charts = '#average' if (staff_access
or instructor_access) else user
kwargs = {
'qualifiers': {
'category': 'video',
},
}
# This returns video descriptors in the order they appear on the course
video_descriptors = videos_problems_in(course)[0]
video_durations = get_info_videos_descriptors(video_descriptors)[2]
video_ids_str = []
course_video_names = []
for descriptor in video_descriptors:
video_ids_str.append(
(course_key.make_usage_key('video',
descriptor.location.name))._to_string())
course_video_names.append(descriptor.display_name_with_default)
if len(video_descriptors) > 0:
first_video_id = course_key.make_usage_key(
'video', video_descriptors[0].location.name)
# Video progress visualization. Video percentage seen total and non-overlapped.
video_names, avg_video_time, video_percentages = get_module_consumption(
user_for_charts, course_key, 'video', 'video_progress')
if avg_video_time != []:
all_video_time_percent = map(truediv, avg_video_time,
video_durations)
all_video_time_percent = [
int(round(x * 100, 0)) for x in all_video_time_percent
]
else:
all_video_time_percent = avg_video_time
column_headers = ['Video', 'Different video time', 'Total video time']
video_prog_json = ready_for_arraytodatatable(column_headers,
video_names,
video_percentages,
all_video_time_percent)
video_names, all_video_time = get_module_consumption(
user_for_charts, course_key, 'video', 'total_time_vid_prob')[0:2]
# Time spent on every video resource
column_headers = ['Video', 'Time watched']
video_distrib_json = ready_for_arraytodatatable(
column_headers, video_names, all_video_time)
# Video events dispersion within video length
scatter_array = get_video_events_info(user_for_charts, first_video_id)
# Repetitions per video intervals
user_for_vid_intervals = '#class_total_times' if user_for_charts == '#average' else user_for_charts
video_intervals_array = get_user_video_intervals(
user_for_vid_intervals, first_video_id)
# Case no videos in course
else:
video_names = None
video_prog_json = simplejson.dumps(None)
video_distrib_json = simplejson.dumps(None)
scatter_array = simplejson.dumps(None)
video_intervals_array = simplejson.dumps(None)
# Time spent on every problem resource
problem_names, time_x_problem = get_module_consumption(
user_for_charts, course_key, 'problem', 'total_time_vid_prob')[0:2]
column_headers = ['Problem', 'Time on problem']
problem_distrib_json = ready_for_arraytodatatable(column_headers,
problem_names,
time_x_problem)
# Daily time spent on video and/or problem resources
video_days, video_daily_time = get_daily_consumption(
user_for_charts, course_key, 'video')
problem_days, problem_daily_time = get_daily_consumption(
user_for_charts, course_key, 'problem')
vid_and_prob_daily_time = join_video_problem_time(video_days,
video_daily_time,
problem_days,
problem_daily_time)
#Analytics visualizations
if staff_access or instructor_access:
# Instructor access
std_sort = get_DB_sort_course_homework(course_key)
# Chapter time
cs, st = get_DB_course_spent_time(course_key, student_id=ALL_STUDENTS)
students_spent_time = chapter_time_to_js(cs, st)
students_grades = get_DB_student_grades(course_key,
student_id=ALL_STUDENTS)
cs, sa = course_accesses = get_DB_course_section_accesses(
course_key, student_id=ALL_STUDENTS)
students_course_accesses = course_accesses_to_js(cs, sa)
students_prob_vid_progress = get_DB_course_video_problem_progress(
course_key, student_id=ALL_STUDENTS)
students_time_schedule = get_DB_time_schedule(course_key,
student_id=ALL_STUDENTS)
else:
# Sort homework
# Chapter time
std_sort = None
cs, st = get_DB_course_spent_time(course_key, user.id)
students_spent_time = chapter_time_to_js(cs, st)
students_grades = get_DB_student_grades(course_key, user.id)
cs, sa = course_accesses = get_DB_course_section_accesses(
course_key, user.id)
students_course_accesses = course_accesses_to_js(cs, sa)
students_time_schedule = get_DB_time_schedule(course_key, user.id)
students_prob_vid_progress = get_DB_course_video_problem_progress(
course_key, user.id)
context = {
'course': course,
'request': request,
'user': user,
'user_id': user.id,
'staff_access': staff_access,
'instructor_access': instructor_access,
'masquerade': masq,
'studio_url': studio_url,
'reverifications': reverifications,
'course_id': course_id,
'students': students_to_js(get_course_students(course_key)),
'visualizations_id': VISUALIZATIONS_ID,
'std_grades_dump': dumps(students_grades),
'sort_std_dump': dumps(std_sort),
'time_dump': dumps(students_spent_time),
'accesses_dump': dumps(students_course_accesses),
'std_time_schedule_dumb': dumps(students_time_schedule),
'vid_prob_prog_dump': dumps(students_prob_vid_progress),
'pass_limit': pass_limit,
'prof_limit': proficiency_limit,
'usernames_in': usernames_in,
'video_names': course_video_names,
'video_ids': video_ids_str,
'video_prog_json': video_prog_json,
'video_distrib_json': video_distrib_json,
'problem_distrib_json': problem_distrib_json,
'video_intervals_array': video_intervals_array,
'vid_and_prob_daily_time': vid_and_prob_daily_time,
'scatter_array': scatter_array,
}
return render_to_response('learning_analytics/learning_analytics.html',
context)
