def single_assessment_level_export(self, **kwargs):
"""
Collect a single assessment level learner data for the ``EnterpriseCustomer`` where data sharing consent is
granted.
Yields a learner assessment data object for each subsection of the course that the learner is enrolled in,
containing:
* ``enterprise_enrollment``: ``EnterpriseCourseEnrollment`` object.
* ``course_id``: The string ID of the course under the enterprise enrollment.
* ``subsection_id``: The string ID of the subsection within the course.
* ``grade``: string grade recorded for the learner in the course.
"""
learner_to_transmit = kwargs.get('learner_to_transmit', None)
TransmissionAudit = kwargs.get('TransmissionAudit', None) # pylint: disable=invalid-name
course_run_id = kwargs.get('course_run_id', None)
grade = kwargs.get('grade', None)
subsection_id = kwargs.get('subsection_id')
enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related(
'enterprise_customer_user'
).filter(
enterprise_customer_user__active=True,
enterprise_customer_user__user_id=learner_to_transmit.id,
course_id=course_run_id,
).order_by('course_id')
# We are transmitting for a single enrollment, so grab just the one.
enterprise_enrollment = enrollment_queryset.first()
already_transmitted = is_already_transmitted(
TransmissionAudit,
enterprise_enrollment.id,
grade,
subsection_id
)
if not (TransmissionAudit and already_transmitted) and LearnerExporter.has_data_sharing_consent(
enterprise_enrollment):
# No caching because we're only fetching one course detail
course_details = self.fetch_course_details(None, course_run_id)
if course_details:
assessment_grade_data = self._collect_assessment_grades_data(enterprise_enrollment)
records = self.get_learner_assessment_data_records(
enterprise_enrollment=enterprise_enrollment,
assessment_grade_data=assessment_grade_data,
)
if records:
# There are some cases where we won't receive a record from the above
# method; right now, that should only happen if we have an Enterprise-linked
# user for the integrated channel, and transmission of that user's
# data requires an upstream user identifier that we don't have (due to a
# failure of SSO or similar). In such a case, `get_learner_data_record`
# would return None, and we'd simply skip yielding it here.
for record in records:
yield record
def assessment_level_transmit(self, exporter, **kwargs):
"""
Send all assessment level grade information under an enterprise enrollment to the integrated channel using the
client.
Args:
exporter: The learner assessment data exporter used to send to the integrated channel.
kwargs: Contains integrated channel-specific information for customized transmission variables.
- app_label: The app label of the integrated channel for whom to store learner data records for.
- model_name: The name of the specific learner data record model to use.
- remote_user_id: The remote ID field name of the learner on the audit model.
"""
TransmissionAudit = apps.get_model( # pylint: disable=invalid-name
app_label=kwargs.get('app_label', 'integrated_channel'),
model_name=kwargs.get('model_name',
'LearnerDataTransmissionAudit'),
)
kwargs.update(TransmissionAudit=TransmissionAudit, )
# Retrieve learner data for each existing enterprise enrollment under the enterprise customer
# and transmit the data according to the current enterprise configuration.
for learner_data in exporter.bulk_assessment_level_export():
serialized_payload = learner_data.serialize(
enterprise_configuration=self.enterprise_configuration)
enterprise_enrollment_id = learner_data.enterprise_course_enrollment_id
# Check the last transmission for the current enrollment and see if the old grades match the new ones
if is_already_transmitted(TransmissionAudit,
enterprise_enrollment_id,
learner_data.grade,
learner_data.subsection_id):
# We've already sent a completion status for this enrollment
LOGGER.info(
'Skipping previously sent enterprise enrollment {}'.format(
enterprise_enrollment_id))
continue
try:
code, body = self.client.create_assessment_reporting(
getattr(learner_data, kwargs.get('remote_user_id')),
serialized_payload)
LOGGER.info(
'Successfully sent completion status call for enterprise enrollment {}'
.format(enterprise_enrollment_id, ))
except ClientError as client_error:
code = client_error.status_code
body = client_error.message
self.handle_transmission_error(learner_data, client_error)
learner_data.status = str(code)
learner_data.error_message = body if code >= 400 else ''
learner_data.save()
def export(self, **kwargs): # pylint: disable=R0915
"""
Collect learner data for the ``EnterpriseCustomer`` where data sharing consent is granted.
Yields a learner data object for each enrollment, containing:
* ``enterprise_enrollment``: ``EnterpriseCourseEnrollment`` object.
* ``completed_date``: datetime instance containing the course/enrollment completion date; None if not complete.
"Course completion" occurs for instructor-paced courses when course certificates are issued, and
for self-paced courses, when the course end date is passed, or when the learner achieves a passing grade.
* ``grade``: string grade recorded for the learner in the course.
"""
channel_name = kwargs.get('app_label')
exporting_single_learner = False
learner_to_transmit = kwargs.get('learner_to_transmit', None)
course_run_id = kwargs.get('course_run_id', None)
completed_date = kwargs.get('completed_date', None)
is_passing = kwargs.get('is_passing', False)
grade = kwargs.get('grade', None)
skip_transmitted = kwargs.get('skip_transmitted', True)
TransmissionAudit = kwargs.get('TransmissionAudit', None) # pylint: disable=invalid-name
# Fetch the consenting enrollment data, including the enterprise_customer_user.
# Order by the course_id, to avoid fetching course API data more than we have to.
generate_formatted_log(
'Starting Export. CompletedDate: {completed_date}, Course: {course_run}, '
'Grade: {grade}, IsPassing: {is_passing}, User: {user_id}'.format(
completed_date=completed_date,
course_run=course_run_id,
grade=grade,
is_passing=is_passing,
user_id=learner_to_transmit.id if learner_to_transmit else None
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
enrollment_queryset = EnterpriseCourseEnrollment.objects.select_related(
'enterprise_customer_user'
).filter(
enterprise_customer_user__enterprise_customer=self.enterprise_customer,
enterprise_customer_user__active=True,
)
if learner_to_transmit and course_run_id:
enrollment_queryset = enrollment_queryset.filter(
course_id=course_run_id,
enterprise_customer_user__user_id=learner_to_transmit.id,
)
exporting_single_learner = True
generate_formatted_log(
'Exporting single learner. Course: {course_run}, User: {user_id}'.format(
course_run=course_run_id,
user_id=learner_to_transmit.id
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
enrollment_queryset = enrollment_queryset.order_by('course_id')
# Fetch course details from the Course API, and cache between calls.
course_details = None
generate_formatted_log(
'Beginning export of enrollments: {enrollments}.'.format(
enrollments=list(enrollment_queryset.values()),
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
for enterprise_enrollment in enrollment_queryset:
is_audit_enrollment = enterprise_enrollment.is_audit_enrollment
if TransmissionAudit and skip_transmitted and \
is_already_transmitted(TransmissionAudit, enterprise_enrollment.id, grade):
# We've already sent a completion status for this enrollment
generate_formatted_log(
'Skipping export of previously sent enterprise enrollment. '
'EnterpriseEnrollment: {enterprise_enrollment_id}'.format(
enterprise_enrollment_id=enterprise_enrollment.id
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
continue
course_id = enterprise_enrollment.course_id
# Fetch course details from Courses API
# pylint: disable=unsubscriptable-object
if course_details:
generate_formatted_log(
'Currently exporting for course: {curr_course}, '
'but course details already found: {course_details}'.format(
curr_course=course_id,
course_details=course_details
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
if course_details is None or course_details['course_id'] != course_id:
if self.course_api is None:
self.course_api = CourseApiClient()
course_details = self.course_api.get_course_details(course_id)
generate_formatted_log(
'Successfully retrieved course details for course: {}'.format(
course_id
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
if course_details is None:
# Course not found, so we have nothing to report.
generate_formatted_log(
'Course run details not found. EnterpriseEnrollment: {enterprise_enrollment_pk}, '
'Course: {course_id}'.format(
enterprise_enrollment_pk=enterprise_enrollment.pk,
course_id=course_id
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name,
is_error=True,
)
continue
if (not LearnerExporter.has_data_sharing_consent(enterprise_enrollment) or
enterprise_enrollment.audit_reporting_disabled):
continue
# For instructor-paced and not audit courses, let the certificate determine course completion
if course_details.get('pacing') == 'instructor' and not is_audit_enrollment:
completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent = \
self._collect_certificate_data(enterprise_enrollment)
generate_formatted_log(
'Received data from certificate api. CompletedDate: {completed_date}, Course: {course_id}, '
'Enterprise: {enterprise}, Grade: {grade}, IsPassing: {is_passing}, User: {user_id}'.format(
completed_date=completed_date_from_api,
grade=grade_from_api,
is_passing=is_passing_from_api,
course_id=course_id,
user_id=enterprise_enrollment.enterprise_customer_user.user_id,
enterprise=enterprise_enrollment.enterprise_customer_user.enterprise_customer.slug
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
# For self-paced courses, check the Grades API
else:
completed_date_from_api, grade_from_api, is_passing_from_api, grade_percent = \
self._collect_grades_data(enterprise_enrollment, course_details, is_audit_enrollment)
generate_formatted_log(
'Received data from grades api. CompletedDate: {completed_date}, Course: {course_id}, '
'Enterprise: {enterprise}, Grade: {grade}, IsPassing: {is_passing}, User: {user_id}'.format(
completed_date=completed_date_from_api,
grade=grade_from_api,
is_passing=is_passing_from_api,
course_id=course_id,
user_id=enterprise_enrollment.enterprise_customer_user.user_id,
enterprise=enterprise_enrollment.enterprise_customer_user.enterprise_customer.slug
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name
)
if exporting_single_learner and (grade != grade_from_api or is_passing != is_passing_from_api):
enterprise_user = enterprise_enrollment.enterprise_customer_user
generate_formatted_log(
'Attempt to transmit conflicting data. '
' Course: {course_id}, Enterprise: {enterprise},'
' EnrollmentId: {enrollment_id},'
' Grade: {grade}, GradeAPI: {grade_api}, IsPassing: {is_passing},'
' IsPassingAPI: {is_passing_api}, User: {user_id}'.format(
grade=grade,
is_passing=is_passing,
grade_api=grade_from_api,
is_passing_api=is_passing_from_api,
course_id=course_id,
enrollment_id=enterprise_enrollment.id,
user_id=enterprise_user.user_id,
enterprise=enterprise_user.enterprise_customer.slug
),
channel_name=channel_name,
enterprise_customer_identifier=self.enterprise_customer.name,
is_error=True
)
# Apply the Single Source of Truth for Grades
grade = grade_from_api
completed_date = completed_date_from_api
is_passing = is_passing_from_api
records = self.get_learner_data_records(
enterprise_enrollment=enterprise_enrollment,
completed_date=completed_date,
grade=grade,
is_passing=is_passing,
grade_percent=grade_percent
)
if records:
# There are some cases where we won't receive a record from the above
# method; right now, that should only happen if we have an Enterprise-linked
# user for the integrated channel, and transmission of that user's
# data requires an upstream user identifier that we don't have (due to a
# failure of SSO or similar). In such a case, `get_learner_data_record`
# would return None, and we'd simply skip yielding it here.
for record in records:
yield record
def transmit(self, payload, **kwargs):
"""
Send a completion status call to the integrated channel using the client.
Args:
payload: The learner completion data payload to send to the integrated channel.
kwargs: Contains integrated channel-specific information for customized transmission variables.
- app_label: The app label of the integrated channel for whom to store learner data records for.
- model_name: The name of the specific learner data record model to use.
- remote_user_id: The remote ID field name of the learner on the audit model.
"""
TransmissionAudit = apps.get_model( # pylint: disable=invalid-name
app_label=kwargs.get('app_label', 'integrated_channel'),
model_name=kwargs.get('model_name',
'LearnerDataTransmissionAudit'),
)
kwargs.update(TransmissionAudit=TransmissionAudit, )
# Since we have started sending courses to integrated channels instead of course runs,
# we need to attempt to send transmissions with course keys and course run ids in order to
# ensure that we account for whether courses or course runs exist in the integrated channel.
# The exporters have been changed to return multiple transmission records to attempt,
# one by course key and one by course run id.
# If the transmission with the course key succeeds, the next one will get skipped.
# If it fails, the one with the course run id will be attempted and (presumably) succeed.
for learner_data in payload.export(**kwargs):
serialized_payload = learner_data.serialize(
enterprise_configuration=self.enterprise_configuration)
LOGGER.debug('Attempting to transmit serialized payload: %s',
serialized_payload)
enterprise_enrollment_id = learner_data.enterprise_course_enrollment_id
if learner_data.completed_timestamp is None:
# The user has not completed the course, so we shouldn't send a completion status call
LOGGER.info(
'Skipping in-progress enterprise enrollment {}'.format(
enterprise_enrollment_id))
continue
grade = getattr(learner_data, 'grade', None)
if is_already_transmitted(TransmissionAudit,
enterprise_enrollment_id, grade):
# We've already sent a completion status for this enrollment
LOGGER.info(
'Skipping previously sent enterprise enrollment {}'.format(
enterprise_enrollment_id))
continue
try:
code, body = self.client.create_course_completion(
getattr(learner_data, kwargs.get('remote_user_id')),
serialized_payload)
LOGGER.info(
'Successfully sent completion status call for enterprise enrollment {}'
.format(enterprise_enrollment_id, ))
except ClientError as client_error:
code = client_error.status_code
body = client_error.message
self.handle_transmission_error(learner_data, client_error)
learner_data.status = str(code)
learner_data.error_message = body if code >= 400 else ''
learner_data.save()