def get_days_to_complete(course_id, date_for):
"""Return a dict with a list of days to complete and errors
NOTE: This is a work in progress, as it has issues to resolve:
* It returns the delta in days, so working in ints
* This means if a learner starts at midnight and finished just before
midnight, then 0 days will be given
NOTE: This has limited scaling. We ought to test it with
1k, 10k, 100k cert records
TODO: change to use start_date, end_date with defaults that
start_date is open and end_date is today
TODO: Consider collecting the total seconds rather than days
This will improve accuracy, but may actually not be that important
TODO: Analyze the error based on number of completions
When we have to support scale, we can look into optimization
techinques.
"""
certificates = GeneratedCertificate.objects.filter(
course_id=as_course_key(course_id),
created_date__lte=as_datetime(date_for))
days = []
errors = []
for cert in certificates:
ce = CourseEnrollment.objects.filter(
course_id=as_course_key(course_id), user=cert.user)
# How do we want to handle multiples?
if ce.count() > 1:
errors.append(
dict(
msg='Multiple CE records',
course_id=course_id,
user_id=cert.user.id,
))
try:
days.append((cert.created_date - ce[0].created).days)
except IndexError:
# sometimes a course enrollment is deleted after the cert is generated. why, who knows?
# in which case just leave out that data
errors.append(
dict(
msg='No CourseEnrollment matching user course certificate',
course_id=course_id,
user_id=cert.user.id,
))
return dict(days=days, errors=errors)
def course_enrollments_for_course(course_id):
"""Return a queryset of all `CourseEnrollment` records for a course
TODO: Update this to require the site
Relies on the fact that course_ids are globally unique
"""
return CourseEnrollment.objects.filter(course_id=as_course_key(course_id))
def seed_course_overviews(data=None):
if not data:
data = cans.COURSE_OVERVIEW_DATA
# append with randomly generated course overviews to test pagination
new_courses = [
generate_course_overview(i, org='FOO') for i in xrange(20)
]
data += new_courses
for rec in data:
course_id = rec['id']
defaults = dict(
display_name=rec['display_name'],
org=rec['org'],
display_org_with_default=rec['org'],
number=rec['number'],
created=as_datetime(rec['created']).replace(tzinfo=utc),
start=as_datetime(rec['enrollment_start']).replace(tzinfo=utc),
end=as_datetime(rec['enrollment_end']).replace(tzinfo=utc),
enrollment_start=as_datetime(
rec['enrollment_start']).replace(tzinfo=utc),
enrollment_end=as_datetime(
rec['enrollment_end']).replace(tzinfo=utc),
)
if RELEASE_LINE != 'ginkgo':
defaults['version'] = CourseOverview.VERSION
CourseOverview.objects.update_or_create(
id=as_course_key(course_id),
defaults=defaults,
)
def _create(cls, model_class, *args, **kwargs):
manager = cls._get_manager(model_class)
course_kwargs = {}
for key in kwargs.keys():
if key.startswith('course__'):
course_kwargs[key.split('__')[1]] = kwargs.pop(key)
if 'course' not in kwargs:
course_id = kwargs.get('course_id')
course_overview = None
if course_id is not None:
if isinstance(course_id, six.string_types):
course_id = as_course_key(course_id)
course_kwargs.setdefault('id', course_id)
try:
course_overview = CourseOverview.get_from_id(course_id)
except CourseOverview.DoesNotExist:
pass
if course_overview is None:
course_overview = CourseOverviewFactory(**course_kwargs)
kwargs['course'] = course_overview
return manager.create(*args, **kwargs)
def test_from_course_locator(self):
course_locator = CourseLocator.from_string(
self.course_key_string)
course_key = as_course_key(course_locator)
assert isinstance(course_key, CourseKey)
assert course_key == self.course_key
assert course_key is course_locator
class StudentModuleFactory(DjangoModelFactory):
class Meta:
model = StudentModule
student = factory.SubFactory(
UserFactory,
)
course_id = factory.Sequence(lambda n: as_course_key(
COURSE_ID_STR_TEMPLATE.format(n)))
created = fuzzy.FuzzyDateTime(datetime.datetime(
2018,2,2, tzinfo=factory.compat.UTC))
modified = fuzzy.FuzzyDateTime(datetime.datetime(
2018,2,2, tzinfo=factory.compat.UTC))
@classmethod
def from_course_enrollment(cls, course_enrollment, **kwargs):
"""Contruct a StudentModule for the given CourseEnrollment
kwargs provides for additional optional parameters if you need to
override the default factory assignment
"""
kwargs.update({
'student': course_enrollment.user,
'course_id': course_enrollment.course_id,
})
return cls(**kwargs)
def populate_single_cdm(course_id,
date_for=None,
ed_next=False,
force_update=False):
"""Populates a CourseDailyMetrics record for the given date and course
The calling function is responsible for error handling calls to this
function
"""
if date_for:
date_for = as_date(date_for)
# Provide info in celery log
learner_count = CourseEnrollment.objects.filter(
course_id=as_course_key(course_id)).count()
msg = 'populate_single_cdm. course id = "{}", learner count={}'.format(
course_id, learner_count)
logger.debug(msg)
start_time = time.time()
cdm_obj, _created = CourseDailyMetricsLoader(course_id).load(
date_for=date_for, ed_next=ed_next, force_update=force_update)
elapsed_time = time.time() - start_time
logger.debug('done. Elapsed time (seconds)={}. cdm_obj={}'.format(
elapsed_time, cdm_obj))
def seed_lcgm_for_course(**_kwargs):
"""Quick hack to create a number of LCGM records
Improvement is to add a devsite model for "synthetic course policy". This
model specifies course info: points possible, sections possible, number of
learners or learer range, learner completion/progress curve
"""
date_for = _kwargs.get('date_for', datetime.datetime.utcnow().date())
site = _kwargs.get('site', get_site())
course_id = _kwargs.get('course_id')
points_possible = _kwargs.get('points_possible', 20)
points_earned = _kwargs.get('points_earned', 10)
sections_possible = _kwargs.get('sections_possible', 10)
sections_worked = _kwargs.get('sections_worked', 5)
for ce in CourseEnrollment.objects.filter(course_id=as_course_key(course_id)):
LearnerCourseGradeMetrics.objects.update_or_create(
site=site,
user=ce.user,
course_id=str(course_id),
date_for=date_for,
defaults=dict(
points_possible=points_possible,
points_earned=points_earned,
sections_possible=sections_possible,
sections_worked=sections_worked
)
)
def test_get_list(self):
'''Tests retrieving a list of users with abbreviated details
The fields in each returned record are identified by
`figures.serializers.UserIndexSerializer`
'''
request = APIRequestFactory().get(self.request_path)
force_authenticate(request, user=self.staff_user)
view = self.view_class.as_view({'get': 'list'})
response = view(request)
# Later, we'll elaborate on the tests. For now, some basic checks
assert response.status_code == 200
assert set(response.data.keys()) == set([
'count',
'next',
'previous',
'results',
])
assert len(response.data['results']) == len(self.course_overviews)
for rec in response.data['results']:
course_overview = CourseOverview.objects.get(
id=as_course_key(rec['course_id']))
# Test top level vars
assert rec['course_name'] == course_overview.display_name
assert rec['course_id'] == str(course_overview.id)
def make_course(**kwargs):
return CourseOverviewFactory(
id=as_course_key(kwargs['id']),
display_name=kwargs['name'],
org=kwargs['org'],
number=kwargs['number'],
)
class CourseAccessRoleFactory(DjangoModelFactory):
class Meta:
model = CourseAccessRole
user = factory.SubFactory(UserFactory, )
course_id = factory.Sequence(
lambda n: as_course_key(COURSE_ID_STR_TEMPLATE.format(n)))
role = factory.Iterator(['instructor', 'staff'])
def get_course_enrollments(course_id, date_for):
"""Convenience method to get a filterd queryset of CourseEnrollment objects
"""
return CourseEnrollment.objects.filter(
course_id=as_course_key(course_id),
created__lt=as_datetime(next_day(date_for)),
)
def get_courses(self, user):
course_ids = CourseEnrollment.objects.filter(
user=user).values_list('course_id', flat=True).distinct()
course_overviews = CourseOverview.objects.filter(
id__in=[as_course_key(course_id) for course_id in course_ids])
return [CourseOverviewSerializer(data).data for data in course_overviews]
class StudentModuleFactory(DjangoModelFactory):
class Meta:
model = StudentModule
student = factory.SubFactory(UserFactory, )
course_id = factory.Sequence(
lambda n: as_course_key(COURSE_ID_STR_TEMPLATE.format(n)))
created = fuzzy.FuzzyDateTime(
datetime.datetime(2018, 02, 02, tzinfo=factory.compat.UTC))
def get_active_learner_ids_today(course_id, date_for):
"""Get unique user ids for learners who are active today for the given
course and date
"""
return StudentModule.objects.filter(
course_id=as_course_key(course_id),
modified=as_datetime(date_for)).values_list('student__id',
flat=True).distinct()
def get_student_modules_for_course_in_site(site, course_id):
if is_multisite():
site_id = site.id
check_site = get_site_for_course(course_id)
if not check_site or site_id != check_site.id:
CourseNotInSiteError(
'course "{}"" does not belong to site "{}"'.format(
course_id, site_id))
return StudentModule.objects.filter(course_id=as_course_key(course_id))
def test_unlinked_course_id_param_invalid(self, monkeypatch, lm_test_data):
"""Test that the 'course' query parameter works
"""
our_courses = lm_test_data['us']['courses']
unlinked_course_id = as_course_key('course-v1:UnlinkedCourse+UMK+1999')
query_params = '?course={}&course={}'.format(str(our_courses[0].id),
unlinked_course_id)
assert self.invalid_course_ids_raise_404(monkeypatch, lm_test_data,
query_params)
def get_course_keys_for_site(site):
if figures.helpers.is_multisite():
orgs = organizations.models.Organization.objects.filter(
sites__in=[site])
org_courses = organizations.models.OrganizationCourse.objects.filter(
organization__in=orgs)
course_ids = org_courses.values_list('course_id', flat=True)
else:
course_ids = CourseOverview.objects.all().values_list('id', flat=True)
return [as_course_key(cid) for cid in course_ids]
def __init__(self, course_id):
"""
Initial version, we pass in a course ID and cast to a course key as an
instance attribute. Later on, add `CourseLike` to abstract course identity
so we can stop worrying about "Is it a string repretentation of a course or
is it a CourseKey?"
"""
self.course_key = as_course_key(course_id)
# Improvement: Consider doing lazy evaluation
self.site = get_site_for_course(self.course_id)
def get_all_mau_for_site_course(site, courselike, month_for):
"""
Extract a queryset of distinct MAU user ids for the site and course
"""
sm_recs = get_student_modules_for_course_in_site(site,
as_course_key(courselike))
mau_ids = get_mau_from_student_modules(student_modules=sm_recs,
year=month_for.year,
month=month_for.month)
return mau_ids
def set_enrollment_data(self,
site,
user,
course_id,
course_enrollment=None):
"""
This is an expensive call as it needs to call CourseGradeFactory if
there is not already a LearnerCourseGradeMetrics record for the learner
"""
if not course_enrollment:
# For now, let it raise a `CourseEnrollment.DoesNotExist
# Later on we can add a try block and raise out own custom
# exception
course_enrollment = CourseEnrollment.objects.get(
user=user, course_id=as_course_key(course_id))
defaults = dict(
is_enrolled=course_enrollment.is_active,
date_enrolled=course_enrollment.created,
)
# Note: doesn't use site for filtering
lcgm = LearnerCourseGradeMetrics.objects.latest_lcgm(
user=user, course_id=str(course_id))
if lcgm:
# do we already have an enrollment data record
# We may change this to use
progress_data = dict(date_for=lcgm.date_for,
is_completed=lcgm.completed,
progress_percent=lcgm.progress_percent,
points_possible=lcgm.points_possible,
points_earned=lcgm.points_earned,
sections_possible=lcgm.sections_possible,
sections_worked=lcgm.sections_worked)
else:
ep = EnrollmentProgress(user=user, course_id=course_id)
# TODO: If we get progress worked and there is no LCGM, then we have
# a bug OR there was progress after the last daily metrics collection
progress_data = dict(
date_for=date.today(),
is_completed=ep.is_completed(),
progress_percent=ep.progress_percent(),
points_possible=ep.progress.get('points_possible', 0),
points_earned=ep.progress.get('points_earned', 0),
sections_possible=ep.progress.get('sections_possible', 0),
sections_worked=ep.progress.get('sections_worked', 0))
defaults.update(progress_data)
obj, created = self.update_or_create(site=site,
user=user,
course_id=str(course_id),
defaults=defaults)
return obj, created
def seed_student_modules_fixed(data=None):
'''
'''
if not data:
data = STUDENT_MODULE_DATA
for rec in data:
StudentModule.objects.update_or_create(
student=get_user_model().objects.get(username=rec['username']),
course_id=as_course_key(rec['course_id']),
create=as_datetime(rec['created']),
modified=as_datetime(rec['modified']),
)
def seed_course_access_roles(data=None):
if not data:
data = cans.COURSE_ACCESS_ROLE_DATA
for rec in data:
print('creating course access role')
CourseAccessRole.objects.update_or_create(
user=get_user_model().objects.get(username=rec['username']),
org=rec['org'],
course_id=as_course_key(rec['course_id']),
role=rec['role'],
)
def get_num_learners_completed(course_id, date_for):
"""
Get the total number of certificates generated for the course up to the
'date_for' date
We will need to relabel this to "certificates"
We may want to get the number of certificates granted in the given day
"""
certificates = GeneratedCertificate.objects.filter(
course_id=as_course_key(course_id),
created_date__lt=as_datetime(next_day(date_for)))
return certificates.count()
class CourseOverviewFactory(DjangoModelFactory):
class Meta:
model = CourseOverview
# Only define the fields that we will retrieve
id = factory.Sequence(
lambda n: as_course_key(COURSE_ID_STR_TEMPLATE.format(n)))
display_name = factory.Sequence(lambda n: 'SFA Course {}'.format(n))
org = 'StarFleetAcademy'
number = '2161'
display_org_with_default = factory.LazyAttribute(lambda o: o.org)
created = fuzzy.FuzzyDateTime(
datetime.datetime(2018, 02, 01, tzinfo=factory.compat.UTC))
def get_course_keys_for_site(site):
"""
Developer note: We could improve this function with caching
Question is which is the most efficient way to know cache expiry
We may also be able to reduce the queries here to also improve performance
"""
if is_multisite():
course_ids = site_course_ids(site)
else:
course_ids = CourseOverview.objects.all().values_list('id', flat=True)
return [as_course_key(cid) for cid in course_ids]
def __init__(self, user_id, course_id, **_kwargs):
"""
If figures.compat.course_grade is unable to retrieve the course blocks,
raises:
django.core.exceptions.PermissionDenied(
"User does not have access to this course")
"""
self.learner = get_user_model().objects.get(id=user_id)
self.course = get_course_by_id(course_key=as_course_key(course_id))
self.course._field_data_cache = {} # pylint: disable=protected-access
self.course.set_grading_policy(self.course.grading_policy)
self.course_grade = course_grade(self.learner, self.course)
def get_active_learner_ids_today(course_id, date_for):
"""Get unique user ids for learners who are active today for the given
course and date
Note: When Figures no longer has to support Django 1.8, we can simplify
this date check:
https://docs.djangoproject.com/en/1.9/ref/models/querysets/#date
"""
date_for_as_datetime = as_datetime(date_for)
return StudentModule.objects.filter(
course_id=as_course_key(course_id),
modified__year=date_for_as_datetime.year,
modified__month=date_for_as_datetime.month,
modified__day=date_for_as_datetime.day,
).values_list('student__id', flat=True).distinct()
class CourseUserGroupFactory(DjangoModelFactory):
class Meta:
model = CourseUserGroup
name = factory.Sequence(lambda n: "CourseTeam #%s" % n)
course_id = factory.Sequence(lambda n: as_course_key(
COURSE_ID_STR_TEMPLATE.format(n)))
group_type = CourseUserGroup.COHORT
@factory.post_generation
def users(self, create, extracted, **kwargs):
if not create:
return
if extracted:
for user in extracted:
self.users.add(user)
def seed_course_enrollments_for_course(course_id, users, max_days_back):
def enroll_date(max_days_back):
days_back = random.randint(1, abs(max_days_back))
return days_from(LAST_DAY, days_back * -1)
for user in users:
if VERBOSE:
print('seeding course enrollment for user {}'.format(
user.username))
CourseEnrollment.objects.update_or_create(
course_id=as_course_key(course_id),
course_overview=CourseOverview.objects.get(id=course_id),
user=user,
created=as_datetime(
enroll_date(max_days_back)).replace(tzinfo=utc),
)
