class Migration(migrations.Migration):
dependencies = [
('courseware', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='StudentModuleHistoryExtended',
fields=[
('version',
models.CharField(db_index=True,
max_length=255,
null=True,
blank=True)),
('created', models.DateTimeField(db_index=True)),
('state', models.TextField(null=True, blank=True)),
('grade', models.FloatField(null=True, blank=True)),
('max_grade', models.FloatField(null=True, blank=True)),
('id',
UnsignedBigIntAutoField(serialize=False, primary_key=True)),
('student_module',
models.ForeignKey(
to='courseware.StudentModule',
on_delete=django.db.models.deletion.DO_NOTHING,
db_constraint=False)),
],
options={
'get_latest_by': 'created',
},
),
migrations.RunPython(bump_pk_start,
reverse_code=migrations.RunPython.noop,
atomic=False),
]
class Migration(migrations.Migration):
dependencies = [
('grades', '0005_multiple_course_flags'),
]
operations = [
migrations.CreateModel(
name='PersistentCourseGrade',
fields=[
('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
('user_id', models.IntegerField(db_index=True)),
('course_id', CourseKeyField(max_length=255)),
('course_edited_timestamp', models.DateTimeField(verbose_name='Last content edit timestamp')),
('course_version', models.CharField(max_length=255, verbose_name='Course content version identifier', blank=True)),
('grading_policy_hash', models.CharField(max_length=255, verbose_name='Hash of grading policy')),
('percent_grade', models.FloatField()),
('letter_grade', models.CharField(max_length=255, verbose_name='Letter grade for course')),
],
),
migrations.AlterUniqueTogether(
name='persistentcoursegrade',
unique_together=set([('course_id', 'user_id')]),
),
]
class StudentModuleHistoryExtended(BaseStudentModuleHistory):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size.
This new extended CSMH has a larger primary key that won't run out of space
so quickly."""
class Meta(object):
app_label = 'coursewarehistoryextended'
get_latest_by = "created"
index_together = ['student_module']
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
student_module = models.ForeignKey(StudentModule,
db_index=True,
db_constraint=False,
on_delete=models.DO_NOTHING)
@receiver(post_save, sender=StudentModule)
def save_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Checks the instance's module_type, and creates & saves a
StudentModuleHistoryExtended entry if the module_type is one that
we save.
"""
if instance.module_type in StudentModuleHistoryExtended.HISTORY_SAVING_TYPES:
history_entry = StudentModuleHistoryExtended(
student_module=instance,
version=None,
created=instance.modified,
state=instance.state,
grade=instance.grade,
max_grade=instance.max_grade)
history_entry.save()
@receiver(post_delete, sender=StudentModule)
def delete_history(sender, instance, **kwargs): # pylint: disable=no-self-argument, unused-argument
"""
Django can't cascade delete across databases, so we tell it at the model level to
on_delete=DO_NOTHING and then listen for post_delete so we can clean up the CSMHE rows.
"""
StudentModuleHistoryExtended.objects.filter(
student_module=instance).all().delete()
def __str__(self):
return six.text_type(repr(self))
class Migration(migrations.Migration):
dependencies = [
]
operations = [
migrations.CreateModel(
name='PersistentSubsectionGrade',
fields=[
('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
('user_id', models.IntegerField()),
('course_id', CourseKeyField(max_length=255)),
('usage_key', UsageKeyField(max_length=255)),
('subtree_edited_date', models.DateTimeField(verbose_name='last content edit timestamp')),
('course_version', models.CharField(max_length=255, verbose_name='guid of latest course version', blank=True)),
('earned_all', models.FloatField()),
('possible_all', models.FloatField()),
('earned_graded', models.FloatField()),
('possible_graded', models.FloatField()),
],
),
migrations.CreateModel(
name='VisibleBlocks',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('blocks_json', models.TextField()),
('hashed', models.CharField(unique=True, max_length=100)),
],
),
migrations.AddField(
model_name='persistentsubsectiongrade',
name='visible_blocks',
field=models.ForeignKey(to='grades.VisibleBlocks', db_column='visible_blocks_hash', to_field='hashed', on_delete=models.CASCADE),
),
migrations.AlterUniqueTogether(
name='persistentsubsectiongrade',
unique_together={('course_id', 'user_id', 'usage_key')},
),
]
class PersistentCourseGrade(TimeStampedModel):
"""
A django model tracking persistent course grades.
.. no_pii:
"""
class Meta:
app_label = "grades"
# Indices:
# (course_id, user_id) for individual grades
# (course_id) for instructors to see all course grades, implicitly created via the unique_together constraint
# (user_id) for course dashboard; explicitly declared as an index below
# (passed_timestamp, course_id) for tracking when users first earned a passing grade.
# (modified): find all the grades updated within a certain timespan
# (modified, course_id): find all the grades updated within a certain timespan for a course
unique_together = [
('course_id', 'user_id'),
]
index_together = [('passed_timestamp', 'course_id'),
('modified', 'course_id')]
# primary key will need to be large for this table
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
user_id = models.IntegerField(blank=False, db_index=True)
course_id = CourseKeyField(blank=False, max_length=255)
# Information relating to the state of content when grade was calculated
course_edited_timestamp = models.DateTimeField(
'Last content edit timestamp', blank=True, null=True)
course_version = models.CharField('Course content version identifier',
blank=True,
max_length=255)
grading_policy_hash = models.CharField('Hash of grading policy',
blank=False,
max_length=255)
# Information about the course grade itself
percent_grade = models.FloatField(blank=False)
letter_grade = models.CharField('Letter grade for course',
blank=False,
max_length=255)
# Information related to course completion
passed_timestamp = models.DateTimeField(
'Date learner earned a passing grade', blank=True, null=True)
_CACHE_NAMESPACE = "grades.models.PersistentCourseGrade"
def __str__(self):
"""
Returns a string representation of this model.
"""
return ', '.join([
f"{type(self).__name__} user: {self.user_id}",
f"course version: {self.course_version}",
f"grading policy: {self.grading_policy_hash}",
f"percent grade: {self.percent_grade}%",
f"letter grade: {self.letter_grade}",
f"passed timestamp: {self.passed_timestamp}",
])
@classmethod
def prefetch(cls, course_id, users):
"""
Prefetches grades for the given users for the given course.
"""
get_cache(cls._CACHE_NAMESPACE)[cls._cache_key(course_id)] = {
grade.user_id: grade
for grade in cls.objects.filter(
user_id__in=[user.id for user in users], course_id=course_id)
}
@classmethod
def clear_prefetched_data(cls, course_key):
"""
Clears prefetched grades for this course from the RequestCache.
"""
get_cache(cls._CACHE_NAMESPACE).pop(cls._cache_key(course_key), None)
@classmethod
def read(cls, user_id, course_id):
"""
Reads a grade from database
Arguments:
user_id: The user associated with the desired grade
course_id: The id of the course associated with the desired grade
Raises PersistentCourseGrade.DoesNotExist if applicable
"""
try:
prefetched_grades = get_cache(
cls._CACHE_NAMESPACE)[cls._cache_key(course_id)]
try:
return prefetched_grades[user_id]
except KeyError:
# user's grade is not in the prefetched dict, so
# assume they have no grade
raise cls.DoesNotExist # lint-amnesty, pylint: disable=raise-missing-from
except KeyError:
# grades were not prefetched for the course, so fetch it
return cls.objects.get(user_id=user_id, course_id=course_id)
@classmethod
def update_or_create(cls, user_id, course_id, **kwargs):
"""
Creates a course grade in the database.
Returns a PersistedCourseGrade object.
"""
passed = kwargs.pop('passed')
if kwargs.get('course_version', None) is None:
kwargs['course_version'] = ""
grade, _ = cls.objects.update_or_create(user_id=user_id,
course_id=course_id,
defaults=kwargs)
if passed and not grade.passed_timestamp:
COURSE_GRADE_PASSED_FIRST_TIME.send(sender=None,
course_id=course_id,
user_id=user_id)
grade.passed_timestamp = now()
grade.save()
cls._emit_grade_calculated_event(grade)
cls._update_cache(course_id, user_id, grade)
return grade
@classmethod
def _update_cache(cls, course_id, user_id, grade):
course_cache = get_cache(cls._CACHE_NAMESPACE).get(
cls._cache_key(course_id))
if course_cache is not None:
course_cache[user_id] = grade
@classmethod
def _cache_key(cls, course_id):
return f"grades_cache.{course_id}"
@staticmethod
def _emit_grade_calculated_event(grade):
events.course_grade_calculated(grade)
class PersistentSubsectionGrade(TimeStampedModel):
"""
A django model tracking persistent grades at the subsection level.
.. no_pii:
"""
class Meta:
app_label = "grades"
unique_together = [
# * Specific grades can be pulled using all three columns,
# * Progress page can pull all grades for a given (course_id, user_id)
# * Course staff can see all grades for a course using (course_id,)
('course_id', 'user_id', 'usage_key'),
]
# Allows querying in the following ways:
# (modified): find all the grades updated within a certain timespan
# (modified, course_id): find all the grades updated within a timespan for a certain course
# (modified, course_id, usage_key): find all the grades updated within a timespan for a subsection
# in a course
# (first_attempted, course_id, user_id): find all attempted subsections in a course for a user
# (first_attempted, course_id): find all attempted subsections in a course for all users
index_together = [('modified', 'course_id', 'usage_key'),
('first_attempted', 'course_id', 'user_id')]
# primary key will need to be large for this table
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
user_id = models.IntegerField(blank=False)
course_id = CourseKeyField(blank=False, max_length=255)
# note: the usage_key may not have the run filled in for
# old mongo courses. Use the full_usage_key property
# instead when you want to use/compare the usage_key.
usage_key = UsageKeyField(blank=False, max_length=255)
# Information relating to the state of content when grade was calculated
subtree_edited_timestamp = models.DateTimeField(
'Last content edit timestamp', blank=True, null=True)
course_version = models.CharField('Guid of latest course version',
blank=True,
max_length=255)
# earned/possible refers to the number of points achieved and available to achieve.
# graded refers to the subset of all problems that are marked as being graded.
earned_all = models.FloatField(blank=False)
possible_all = models.FloatField(blank=False)
earned_graded = models.FloatField(blank=False)
possible_graded = models.FloatField(blank=False)
# timestamp for the learner's first attempt at content in
# this subsection. If null, indicates no attempt
# has yet been made.
first_attempted = models.DateTimeField(null=True, blank=True)
# track which blocks were visible at the time of grade calculation
visible_blocks = models.ForeignKey(VisibleBlocks,
db_column='visible_blocks_hash',
to_field='hashed',
on_delete=models.CASCADE)
_CACHE_NAMESPACE = 'grades.models.PersistentSubsectionGrade'
@property
def full_usage_key(self):
"""
Returns the "correct" usage key value with the run filled in.
"""
if self.usage_key.run is None: # lint-amnesty, pylint: disable=no-member
# pylint: disable=unexpected-keyword-arg,no-value-for-parameter
return self.usage_key.replace(course_key=self.course_id)
else:
return self.usage_key
def __str__(self):
"""
Returns a string representation of this model.
"""
return (
"{} user: {}, course version: {}, subsection: {} ({}). {}/{} graded, {}/{} all, first_attempted: {}"
).format(
type(self).__name__,
self.user_id,
self.course_version,
self.usage_key,
self.visible_blocks_id,
self.earned_graded,
self.possible_graded,
self.earned_all,
self.possible_all,
self.first_attempted,
)
@classmethod
def prefetch(cls, course_key, users):
"""
Prefetches grades for the given users in the given course.
"""
cache_key = cls._cache_key(course_key)
get_cache(cls._CACHE_NAMESPACE)[cache_key] = defaultdict(list)
cached_grades = get_cache(cls._CACHE_NAMESPACE)[cache_key]
queryset = cls.objects.select_related(
'visible_blocks', 'override').filter(
user_id__in=[user.id for user in users],
course_id=course_key,
)
for record in queryset:
cached_grades[record.user_id].append(record)
@classmethod
def clear_prefetched_data(cls, course_key):
"""
Clears prefetched grades for this course from the RequestCache.
"""
get_cache(cls._CACHE_NAMESPACE).pop(cls._cache_key(course_key), None)
@classmethod
def read_grade(cls, user_id, usage_key):
"""
Reads a grade from database
Arguments:
user_id: The user associated with the desired grade
usage_key: The location of the subsection associated with the desired grade
Raises PersistentSubsectionGrade.DoesNotExist if applicable
"""
return cls.objects.select_related('visible_blocks', 'override').get(
user_id=user_id,
course_id=usage_key.
course_key, # course_id is included to take advantage of db indexes
usage_key=usage_key,
)
@classmethod
def bulk_read_grades(cls, user_id, course_key):
"""
Reads all grades for the given user and course.
Arguments:
user_id: The user associated with the desired grades
course_key: The course identifier for the desired grades
"""
try:
prefetched_grades = get_cache(
cls._CACHE_NAMESPACE)[cls._cache_key(course_key)]
try:
return prefetched_grades[user_id]
except KeyError:
# The user's grade is not in the cached dict of subsection grades,
# so return an empty list.
return []
except KeyError:
# subsection grades were not prefetched for the course, so get them from the DB
return cls.objects.select_related('visible_blocks',
'override').filter(
user_id=user_id,
course_id=course_key,
)
@classmethod
def update_or_create_grade(cls, **params):
"""
Wrapper for objects.update_or_create.
"""
cls._prepare_params(params)
VisibleBlocks.cached_get_or_create(params['user_id'],
params['visible_blocks'])
cls._prepare_params_visible_blocks_id(params)
# TODO: do we NEED to pop these?
first_attempted = params.pop('first_attempted')
user_id = params.pop('user_id')
usage_key = params.pop('usage_key')
grade, _ = cls.objects.update_or_create(
user_id=user_id,
course_id=usage_key.course_key,
usage_key=usage_key,
defaults=params,
)
# TODO: Remove as part of EDUCATOR-4602.
if str(usage_key.course_key) == 'course-v1:UQx+BUSLEAD5x+2T2019':
log.info(
'Created/updated grade ***{}*** for user ***{}*** in course ***{}***'
'for subsection ***{}*** with default params ***{}***'.format(
grade, user_id, usage_key.course_key, usage_key, params))
grade.override = PersistentSubsectionGradeOverride.get_override(
user_id, usage_key)
if first_attempted is not None and grade.first_attempted is None:
grade.first_attempted = first_attempted
grade.save()
cls._emit_grade_calculated_event(grade)
return grade
@classmethod
def bulk_create_grades(cls, grade_params_iter, user_id, course_key):
"""
Bulk creation of grades.
"""
if not grade_params_iter:
return
PersistentSubsectionGradeOverride.prefetch(user_id, course_key)
list(map(cls._prepare_params, grade_params_iter))
VisibleBlocks.bulk_get_or_create(
user_id, course_key,
[params['visible_blocks'] for params in grade_params_iter])
list(map(cls._prepare_params_visible_blocks_id, grade_params_iter))
grades = [
PersistentSubsectionGrade(**params) for params in grade_params_iter
]
grades = cls.objects.bulk_create(grades)
for grade in grades:
cls._emit_grade_calculated_event(grade)
return grades
@classmethod
def _prepare_params(cls, params):
"""
Prepares the fields for the grade record.
"""
if not params.get('course_id', None):
params['course_id'] = params['usage_key'].course_key
params['course_version'] = params.get('course_version', None) or ""
params['visible_blocks'] = BlockRecordList.from_list(
params['visible_blocks'], params['course_id'])
@classmethod
def _prepare_params_visible_blocks_id(cls, params):
"""
Prepares the visible_blocks_id field for the grade record,
using the hash of the visible_blocks field. Specifying
the hashed field eliminates extra queries to get the
VisibleBlocks record. Use this variation of preparing
the params when you are sure of the existence of the
VisibleBlock.
"""
params['visible_blocks_id'] = params['visible_blocks'].hash_value
del params['visible_blocks']
@staticmethod
def _emit_grade_calculated_event(grade):
events.subsection_grade_calculated(grade)
@classmethod
def _cache_key(cls, course_id):
return f"subsection_grades_cache.{course_id}"
class StudentModule(models.Model):
"""
Keeps student state for a particular XBlock usage and particular student.
Called Module since it was originally used for XModule state.
.. no_pii:
"""
objects = ChunkingManager()
id = UnsignedBigIntAutoField(primary_key=True) # pylint: disable=invalid-name
## The XBlock/XModule type (e.g. "problem")
module_type = models.CharField(max_length=32, db_index=True)
# Key used to share state. This is the XBlock usage_id
module_state_key = UsageKeyField(max_length=255, db_column='module_id')
student = models.ForeignKey(User,
db_index=True,
db_constraint=False,
on_delete=models.CASCADE)
# The learning context of the usage_key (usually a course ID, but may be a library or something else)
course_id = LearningContextKeyField(max_length=255, db_index=True)
class Meta:
app_label = "courseware"
unique_together = (('student', 'module_state_key', 'course_id'), )
indexes = [
models.Index(fields=['module_state_key', 'grade', 'student'],
name="courseware_stats")
]
# Internal state of the object
state = models.TextField(null=True, blank=True)
# Grade, and are we done?
grade = models.FloatField(null=True, blank=True, db_index=True)
max_grade = models.FloatField(null=True, blank=True)
DONE_TYPES = (
('na', 'NOT_APPLICABLE'),
('f', 'FINISHED'),
('i', 'INCOMPLETE'),
)
done = models.CharField(max_length=8, choices=DONE_TYPES, default='na')
created = models.DateTimeField(auto_now_add=True, db_index=True)
modified = models.DateTimeField(auto_now=True, db_index=True)
@classmethod
def all_submitted_problems_read_only(cls, course_id):
"""
Return all model instances that correspond to problems that have been
submitted for a given course. So module_type='problem' and a non-null
grade. Use a read replica if one exists for this environment.
"""
queryset = cls.objects.filter(course_id=course_id,
module_type='problem',
grade__isnull=False)
if "read_replica" in settings.DATABASES:
return queryset.using("read_replica")
else:
return queryset
def __repr__(self):
return 'StudentModule<{!r}>'.format({
'course_id': self.course_id,
'module_type': self.module_type,
# We use the student_id instead of username to avoid a database hop.
# This can actually matter in cases where we're logging many of
# these (e.g. on a broken progress page).
'student_id': self.student_id,
'module_state_key': self.module_state_key,
'state': str(self.state)[:20],
})
def __str__(self):
return str(repr(self))
@classmethod
def get_state_by_params(cls,
course_id,
module_state_keys,
student_id=None):
"""
Return all model instances that correspond to a course and module keys.
Student ID is optional keyword argument, if provided it narrows down the instances.
"""
module_states = cls.objects.filter(
course_id=course_id, module_state_key__in=module_state_keys)
if student_id:
module_states = module_states.filter(student_id=student_id)
return module_states
@classmethod
def save_state(cls, student, course_id, module_state_key, defaults): # lint-amnesty, pylint: disable=missing-function-docstring
if not student.is_authenticated:
return
else:
cls.objects.update_or_create(
student=student,
course_id=course_id,
module_state_key=module_state_key,
defaults=defaults,
)