def offline_grade_calculation(course_key):
'''
Compute grades for all students for a specified course, and save results to the DB.
'''
tstart = time.time()
enrolled_students = User.objects.filter(
courseenrollment__course_id=course_key,
courseenrollment__is_active=1
).prefetch_related("groups").order_by('username')
enc = MyEncoder()
print "{} enrolled students".format(len(enrolled_students))
course = get_course_by_id(course_key)
for student in enrolled_students:
request = DummyRequest()
request.user = student
request.session = {}
gradeset = grades.grade(student, request, course, keep_raw_scores=True)
gs = enc.encode(gradeset)
ocg, _created = models.OfflineComputedGrade.objects.get_or_create(user=student, course_id=course_key)
ocg.gradeset = gs
ocg.save()
print "%s done" % student # print statement used because this is run by a management command
tend = time.time()
dt = tend - tstart
ocgl = models.OfflineComputedGradeLog(course_id=course_key, seconds=dt, nstudents=len(enrolled_students))
ocgl.save()
print ocgl
print "All Done!"
def offline_grade_calculation(course_id):
'''
Compute grades for all students for a specified course, and save results to the DB.
'''
tstart = time.time()
enrolled_students = User.objects.filter(
courseenrollment__course_id=course_id,
courseenrollment__is_active=1
).prefetch_related("groups").order_by('username')
enc = MyEncoder()
print "%d enrolled students" % len(enrolled_students)
course = get_course_by_id(course_id)
for student in enrolled_students:
request = DummyRequest()
request.user = student
request.session = {}
gradeset = grades.grade(student, request, course, keep_raw_scores=True)
gs = enc.encode(gradeset)
ocg, created = models.OfflineComputedGrade.objects.get_or_create(user=student, course_id=course_id)
ocg.gradeset = gs
ocg.save()
print "%s done" % student # print statement used because this is run by a management command
tend = time.time()
dt = tend - tstart
ocgl = models.OfflineComputedGradeLog(course_id=course_id, seconds=dt, nstudents=len(enrolled_students))
ocgl.save()
print ocgl
print "All Done!"
def handle(self, *args, **options):
# current grading logic and data schema doesn't handle dates
# datetime.strptime("21/11/06 16:30", "%m/%d/%y %H:%M")
print "args = ", args
course_id = 'MITx/8.01rq_MW/Classical_Mechanics_Reading_Questions_Fall_2012_MW_Section'
fn = "grades.csv"
get_raw_scores = False
if len(args) > 0:
course_id = args[0]
if len(args) > 1:
fn = args[1]
if len(args) > 2:
get_raw_scores = args[2].lower() == 'raw'
request = DummyRequest()
# parse out the course into a coursekey
try:
course_key = CourseKey.from_string(course_id)
# if it's not a new-style course key, parse it from an old-style
# course key
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(
course_id)
try:
course = get_course_by_id(course_key)
# Ok with catching general exception here because this is run as a management command
# and the exception is exposed right away to the user.
except Exception as err: # pylint: disable=broad-except
print "-----------------------------------------------------------------------------"
print "Sorry, cannot find course with id {}".format(course_id)
print "Got exception {}".format(err)
print "Please provide a course ID or course data directory name, eg content-mit-801rq"
return
print "-----------------------------------------------------------------------------"
print "Dumping grades from {} to file {} (get_raw_scores={})".format(
course.id, fn, get_raw_scores)
datatable = get_student_grade_summary_data(
request, course, get_raw_scores=get_raw_scores)
fp = open(fn, 'w')
writer = csv.writer(fp,
dialect='excel',
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerow(
[unicode(s).encode('utf-8') for s in datatable['header']])
for datarow in datatable['data']:
encoded_row = [unicode(s).encode('utf-8') for s in datarow]
writer.writerow(encoded_row)
fp.close()
print "Done: {} records dumped".format(len(datatable['data']))
def test_grade_summary_data_use_offline(self):
"""
Test grade summary data report generation with use_offline True.
"""
request = DummyRequest()
self.answer_question()
data = get_student_grade_summary_data(
request, self.course, use_offline=True)
expected_data = self.get_expected_grade_data(use_offline=True)
self.compare_data(data, expected_data)
def test_grade_summary_data_raw_scores(self):
"""
Test grade summary data report generation with get_raw_scores True.
"""
request = DummyRequest()
self.answer_question()
data = get_student_grade_summary_data(
request, self.course, get_raw_scores=True,
)
expected_data = self.get_expected_grade_data(get_raw_scores=True)
self.compare_data(data, expected_data)
def offline_grade_calculation(course_key):
'''
Compute grades for all students for a specified course, and save results to the DB.
'''
tstart = time.time()
enrolled_students = User.objects.filter(
courseenrollment__course_id=course_key,
courseenrollment__is_active=1
).prefetch_related("groups").order_by('username')
enc = MyEncoder()
print "{} enrolled students".format(len(enrolled_students))
course = get_course_by_id(course_key)
for student in enrolled_students:
request = DummyRequest()
request.user = student
request.session = {}
gradeset = grades.grade(student, request, course, keep_raw_scores=True)
# Convert Score namedtuples to dicts:
totaled_scores = gradeset['totaled_scores']
for section in totaled_scores:
totaled_scores[section] = [score._asdict() for score in totaled_scores[section]]
gradeset['raw_scores'] = [score._asdict() for score in gradeset['raw_scores']]
# Encode as JSON and save:
gradeset_str = enc.encode(gradeset)
ocg, _created = models.OfflineComputedGrade.objects.get_or_create(user=student, course_id=course_key)
ocg.gradeset = gradeset_str
ocg.save()
print "%s done" % student # print statement used because this is run by a management command
tend = time.time()
dt = tend - tstart
ocgl = models.OfflineComputedGradeLog(course_id=course_key, seconds=dt, nstudents=len(enrolled_students))
ocgl.save()
print ocgl
print "All Done!"
def offline_grade_calculation(course):
'''
Compute grades for all students for a specified course, and save results to the DB.
'''
tstart = time.time()
enrolled_students = User.objects.filter(
courseenrollment__course_id=course.id,
courseenrollment__is_active=1).prefetch_related("groups").order_by(
'username')
enc = MyEncoder()
all_studnets = len(enrolled_students)
print "{} enrolled students in {}".format(all_studnets, course.id)
counter = 0
for student in enrolled_students:
counter += 1
if counter % 1000 == 0:
print "{}/{} done: Course {}".format(counter, all_studnets,
course.id)
request = DummyRequest()
request.user = student
request.session = {}
gradeset = grades.grade(student, request, course, keep_raw_scores=True)
ocg, _created = models.OfflineComputedGrade.objects.get_or_create(
user=student, course_id=course.id)
ocg.gradeset = gradeset['grade']
ocg.save()
tend = time.time()
dt = tend - tstart
ocgl = models.OfflineComputedGradeLog(course_id=course.id,
seconds=dt,
nstudents=len(enrolled_students))
ocgl.save()
print ocgl
print "All Done!"
def test_grade_summary_data_defaults(self):
"""
Test grade summary data report generation with all default kwargs.
This test compares the output of the get_student_grade_summary_data
with a dictionary of exected values. The purpose of this test is
to ensure that future changes to the get_student_grade_summary_data
function (for example, mitocw/edx-platform #95).
"""
request = DummyRequest()
self.answer_question()
data = get_student_grade_summary_data(request, self.course)
expected_data = self.get_expected_grade_data()
self.compare_data(data, expected_data)
def test_grade_summary_data(self):
"""
Test grade summary data report generation
"""
self.answer_question()
request = DummyRequest()
data = get_student_grade_summary_data(request, self.course, get_raw_scores=False)
expected_data = {
'students': [self.student_user, self.student_user2],
'header': [
u'ID', u'Username', u'Full Name', u'edX email', u'External email',
u'HW 01', u'HW 02', u'HW 03', u'HW 04', u'HW 05', u'HW 06', u'HW 07',
u'HW 08', u'HW 09', u'HW 10', u'HW 11', u'HW 12', u'HW Avg', u'Lab 01',
u'Lab 02', u'Lab 03', u'Lab 04', u'Lab 05', u'Lab 06', u'Lab 07',
u'Lab 08', u'Lab 09', u'Lab 10', u'Lab 11', u'Lab 12', u'Lab Avg', u'Midterm',
u'Final'
],
'data': [
[
1, u'u1', u'username', u'*****@*****.**', '', 0.0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
],
[
2, u'u2', u'username', u'*****@*****.**', '', 0.3333333333333333, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0.03333333333333333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0
]
],
'assignments': [
u'HW 01', u'HW 02', u'HW 03', u'HW 04', u'HW 05', u'HW 06', u'HW 07', u'HW 08',
u'HW 09', u'HW 10', u'HW 11', u'HW 12', u'HW Avg', u'Lab 01', u'Lab 02',
u'Lab 03', u'Lab 04', u'Lab 05', u'Lab 06', u'Lab 07', u'Lab 08', u'Lab 09',
u'Lab 10', u'Lab 11', u'Lab 12', u'Lab Avg', u'Midterm', u'Final'
]
}
for key in ['assignments', 'header']:
self.assertListEqual(expected_data[key], data[key])
for index, student in enumerate(expected_data['students']):
self.assertEqual(
student.username,
data['students'][index].username
)
self.assertListEqual(
expected_data['data'][index],
data['data'][index]
)
def test_grade_summary_data_no_grades(self):
"""
Test grade summary data report generation with
get_grades set to False.
"""
request = DummyRequest()
self.answer_question()
data = get_student_grade_summary_data(
request, self.course, get_grades=False
)
expected_data = self.get_expected_grade_data(get_grades=False)
# if get_grades == False, get_expected_grade_data does not
# add an "assignments" key.
self.assertNotIn("assignments", expected_data)
self.compare_data(data, expected_data)
def handle(self, *args, **options):
# current grading logic and data schema doesn't handle dates
# datetime.strptime("21/11/06 16:30", "%m/%d/%y %H:%M")
print "args = ", args
course_id = 'MITx/8.01rq_MW/Classical_Mechanics_Reading_Questions_Fall_2012_MW_Section'
fn = "grades.csv"
get_raw_scores = False
if len(args) > 0:
course_id = args[0]
if len(args) > 1:
fn = args[1]
if len(args) > 2:
get_raw_scores = args[2].lower() == 'raw'
request = DummyRequest()
try:
course = get_course_by_id(course_id)
except Exception:
if course_id in modulestore().courses:
course = modulestore().courses[course_id]
else:
print "-----------------------------------------------------------------------------"
print "Sorry, cannot find course %s" % course_id
print "Please provide a course ID or course data directory name, eg content-mit-801rq"
return
print "-----------------------------------------------------------------------------"
print "Dumping grades from %s to file %s (get_raw_scores=%s)" % (course.id, fn, get_raw_scores)
datatable = get_student_grade_summary_data(request, course, course.id, get_raw_scores=get_raw_scores)
fp = open(fn, 'w')
writer = csv.writer(fp, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
writer.writerow(datatable['header'])
for datarow in datatable['data']:
encoded_row = [unicode(s).encode('utf-8') for s in datarow]
writer.writerow(encoded_row)
fp.close()
print "Done: %d records dumped" % len(datatable['data'])
def test_grade_summary_data_get_score_max(self):
"""
Test grade summary data report generation with get_score_max set
to True (also requires get_raw_scores to be True).
"""
request = DummyRequest()
self.answer_question()
data = get_student_grade_summary_data(
request,
self.course,
use_offline=True,
get_raw_scores=True,
get_score_max=True,
)
expected_data = self.get_expected_grade_data(
use_offline=True,
get_raw_scores=True,
get_score_max=True,
)
self.compare_data(data, expected_data)
def cmc_course_completion_report(
upload=CMC_COURSE_COMPLETION_S3_UPLOAD,
store_local=CMC_COURSE_COMPLETION_STORE_LOCAL):
# from celery.contrib import rdb; rdb.set_trace() # celery remote debugger
request = DummyRequest()
dt = str(datetime.now())
dt_date_only = dt.split(' ')[0].replace('-', '')
fn = '/tmp/cmc_course_completion_{0}.csv'.format(
dt.replace(' ', '').replace(':', '-'))
fp = open(fn, 'w')
writer = csv.writer(fp,
dialect='excel',
quotechar='"',
quoting=csv.QUOTE_ALL)
mongo_courses = modulestore().get_courses()
writer.writerow([
'Training/CMC Username', 'Organization', 'Training Email',
'Training Name', 'Job Title', 'Course Title', 'Course Selection',
'Completion Date', 'Last Section Completed'
])
for course in mongo_courses:
if course.org != 'cmc':
continue
get_raw_scores = False
datatable = get_student_grade_summary_data(
request, course, get_raw_scores=get_raw_scores)
for d in datatable['data']:
user_id = d[0]
user = User.objects.get(id=user_id)
profile = UserProfile.objects.get(user=user_id)
# exclude beta-testers...
try:
if 'beta_testers' in CourseAccessRole.objects.get(
user=user, course_id=course.id).role:
continue
except:
pass
# and certain users and email domains
if user.email.lower().split('@')[1] in ('intersystems.com', 'appsembler.com', 'j2interactive.com') or \
user.email.lower() in ('*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**'):
continue
try:
job_title = json.loads(profile.meta)['job-title']
except (KeyError, ValueError):
job_title = ''
try:
organization = json.loads(profile.meta)['organization']
except (KeyError, ValueError):
organization = ''
try:
enroll_date = CourseEnrollment.objects.get(
user=user, course_id=course.id).created
except ItemNotFoundError:
continue
try:
# these are all ungraded courses and we are counting anything with a GeneratedCertificate
# record here as complete.
completion_date = str(
GeneratedCertificate.objects.get(
user=user, course_id=course.id).created_date)
except GeneratedCertificate.DoesNotExist:
completion_date = 'n/a'
try:
smod = StudentModule.objects.filter(
student=user, course_id=course.id,
module_type='chapter').order_by('-created')[0]
mod = modulestore().get_item(smod.module_state_key)
last_section_completed = mod.display_name
except (IndexError, ItemNotFoundError):
last_section_completed = 'n/a'
output_data = [
d[1], organization, user.email, d[2], job_title,
course.display_name,
str(enroll_date),
str(completion_date), last_section_completed
]
encoded_row = [unicode(s).encode('utf-8') for s in output_data]
# writer.writerow(output_data)
writer.writerow(encoded_row)
fp.close()
# email it to specified recipients
try:
fp = open(fn, 'r')
fp.seek(0)
dest_addr = CMC_REPORT_RECIPIENTS
subject = "Nightly CMC course completion status for {0}".format(
dt_date_only)
message = "See attached CSV file"
mail = EmailMessage(subject, message, to=dest_addr)
mail.attach(fp.name, fp.read(), 'text/csv')
mail.send()
except:
logger.warn('CMC Nightly course completion report failed')
finally:
fp.close()
if store_local:
store_dir = CMC_COURSE_COMPLETION_LOCAL_STORAGE_DIR
store_fn = 'cmc_course_completion_{}.csv'.format(dt_date_only)
do_store_local(fn, store_dir, store_fn)
# upload to S3 bucket
if upload:
latest_fn = 'cmc_course_completion_latest.csv'
bucketname = CMC_COURSE_COMPLETION_BUCKET
s3_path_prefix = CMC_COURSE_COMPLETION_S3_FOLDER
do_store_s3(fn, latest_fn, bucketname, s3_path_prefix)
def isc_course_participation_report(
upload=ISC_COURSE_PARTICIPATION_S3_UPLOAD,
store_local=ISC_COURSE_PARTICIPATION_STORE_LOCAL):
"""
Generate an Excel-format CSV report with the following fields for
all users/courses in the system
edX Username, user active/inactive, organization, email, name
job title, course title, course id, course state (draft/published/),
course enrollment date/time, course completion date/time,
course last section completed, course last access date/time
set upload to False if you do not want to upload to S3,
this will also keep temporary files that are created.
"""
request = DummyRequest()
dt = str(datetime.now()).replace(' ', '').replace(':', '-')
fn = '/tmp/isc_course_participation_{0}.csv'.format(dt)
fp = open(fn, 'w')
writer = csv.writer(fp,
dialect='excel',
quotechar='"',
quoting=csv.QUOTE_ALL)
mongo_courses = modulestore().get_courses()
writer.writerow([
'Training Username',
'User Active/Inactive',
'Organization',
'Training Email',
'Training Name',
'Job Title',
'Course Title',
'Course Id',
'Course Org',
'Course Number',
'Course Run',
'Course Visibility',
'Course State',
'Course Enrollment Date',
'Course Completion Date',
'Course Last Section Completed',
'Course Last Access Date',
'Grade',
])
for course in mongo_courses:
datatable = get_student_grade_summary_data(request,
course,
get_grades=True,
get_raw_scores=False)
# dummy for now
visible = course.catalog_visibility in ('None',
'About') and False or True
# course.ispublic is being used in a non-standard way so not reliable as to public visibility
# public = course.ispublic != False
staff_only = course.visible_to_staff_only
course_visibility = (visible
and not staff_only) and 'Public' or 'Private'
course_state = course.has_ended() and 'Ended' or (
course.has_started() and 'Active' or 'Not started')
for d in datatable['data']:
user_id = d[0]
user = User.objects.get(id=user_id)
try:
enrollment = CourseEnrollment.objects.filter(
user_id=user_id, course_id=course.id)[0]
except CourseEnrollment.DoesNotExist:
continue
active = enrollment.is_active and 'active' or 'inactive'
profile = UserProfile.objects.get(user=user_id)
# exclude beta-testers...
try:
if 'beta_testers' in CourseAccessRole.objects.get(
user=user, course_id=course.id).role:
continue
except:
pass
try:
job_title = json.loads(profile.meta)['job-title']
except (KeyError, ValueError):
job_title = ''
try:
organization = json.loads(profile.meta)['organization']
except (KeyError, ValueError):
organization = ''
enroll_date = enrollment.created.astimezone(
tz.gettz('America/New_York'))
try:
# these are all ungraded courses and we are counting anything with a GeneratedCertificate
# record here as complete.
completion_date = str(
GeneratedCertificate.objects.get(
user=user,
course_id=course.id).created_date.astimezone(
tz.gettz('America/New_York')))
except GeneratedCertificate.DoesNotExist:
completion_date = 'n/a'
try:
smod = StudentModule.objects.filter(
student=user, course_id=course.id).order_by('-created')[0]
# smod_ch = StudentModule.objects.filter(student=user, course_id=course.id, module_type='chapter').order_by('-created')[0]
mod = modulestore().get_item(smod.module_state_key)
last_section_completed = mod.display_name
last_access_date = smod.created.astimezone(
tz.gettz('America/New_York'))
except (IndexError, ItemNotFoundError):
last_access_date = 'n/a'
last_section_completed = 'n/a'
try:
grade = d[5]
grade # pyflakes
except IndexError:
# for some reason sometimes a grade isn't calculated. Use a 0.0 here.
d.append(0.0)
output_data = [
d[1], active, organization, user.email, d[2], job_title,
course.display_name,
str(course.id), course.org, course.number, course.location.run,
course_visibility, course_state,
str(enroll_date),
str(completion_date), last_section_completed,
str(last_access_date), d[5]
]
encoded_row = [unicode(s).encode('utf-8') for s in output_data]
# writer.writerow(output_data)
writer.writerow(encoded_row)
fp.close()
# email it to specified recipients
try:
fp = open(fn, 'r')
fp.seek(0)
dest_addr = ISC_COURSE_PARTICIPATION_REPORT_RECIPIENTS
subject = "All course participation status for {0}".format(dt)
message = "See attached CSV file"
mail = EmailMessage(subject, message, to=dest_addr)
mail.attach(fp.name, fp.read(), 'text/csv')
mail.send()
except:
logger.warn('All course participation report failed')
finally:
fp.close()
# overwrite latest on local filesystem
if store_local:
store_dir = ISC_COURSE_PARTICIPATION_LOCAL_STORAGE_DIR
store_fn = 'isc_course_participation.csv'
do_store_local(fn, store_dir, store_fn)
# upload to S3 bucket
if upload:
latest_fn = 'isc_course_participation.csv'
bucketname = ISC_COURSE_PARTICIPATION_BUCKET
s3_path_prefix = ISC_COURSE_COMPLETION_S3_FOLDER
do_store_s3(fn, latest_fn, bucketname, s3_path_prefix)