def export_exam_profiles(self):
"""
Sync any outstanding profiles
"""
if not settings.FEATURES.get("PEARSON_EXAMS_SYNC", False):
return
exam_profiles = ExamProfile.objects.filter(
status=ExamProfile.PROFILE_PENDING).select_related('profile')
file_prefix = now_in_utc().strftime(PEARSON_CDD_FILE_PREFIX)
valid_exam_profiles = []
for exam_profile in exam_profiles:
validated = validate_profile(exam_profile.profile)
if validated:
valid_exam_profiles.append(exam_profile)
else:
exam_profile.status = ExamProfile.PROFILE_INVALID
exam_profile.save()
# write the file out locally
# this will be written out to a file like: /tmp/cdd-20160405_kjfiamdf.dat
with tempfile.NamedTemporaryFile(
prefix=file_prefix,
encoding=PEARSON_FILE_ENCODING,
suffix=PEARSON_FILE_EXTENSION,
mode='w',
) as tsv:
cdd_writer = writers.CDDWriter()
valid_profiles, invalid_profiles = cdd_writer.write(
tsv, valid_exam_profiles)
# flush data to disk before upload
tsv.flush()
try:
audit.ExamDataAuditor().audit_request_file(tsv.name)
except ImproperlyConfigured:
log.exception('Exam auditing improperly configured')
return
except: # pylint: disable=bare-except
log.exception('Unexpected error auditing CDD file')
return
try:
# upload to SFTP server
upload.upload_tsv(tsv.name)
except ImproperlyConfigured:
log.exception(
'export_exam_profiles is improperly configured, please review require settings.'
)
except RetryableSFTPException as exc:
log.exception(
'Retryable error during upload of CDD file to Pearson SFTP')
# retry up to 3 times w/ exponential backoff if this was a connection error
self.retry(exc=exc,
countdown=exponential_backoff(self.request.retries))
except: # pylint: disable=bare-except
log.exception('Unexpected exception uploading CDD file')
return
valid_profile_ids = [exam_profile.id for exam_profile in valid_profiles]
invalid_profile_ids = [
exam_profile.id for exam_profile in invalid_profiles
]
# if this transaction fails, we log it but allow the task to complete
# since the records never got updated, the next run of this task will attempt to reconile those again
# worst-case this means we send duplicate requests to Pearson, but they are idempotent so that's ok
try:
with transaction.atomic():
# update records to reflect the successful upload
if valid_profile_ids:
ExamProfile.objects.filter(id__in=valid_profile_ids).update(
status=ExamProfile.PROFILE_IN_PROGRESS)
# update records to reflect invalid profile
if invalid_profile_ids:
ExamProfile.objects.filter(id__in=invalid_profile_ids).update(
status=ExamProfile.PROFILE_INVALID)
except: # pylint: disable=bare-except
log.exception('Unexpected exception updating ExamProfile.status')
def auditor(valid_settings): # pylint: disable=unused-argument
"""
ExamDataAuditor that cleans up the GPG keys after itself
"""
yield audit.ExamDataAuditor(store=Mock(spec=audit.S3AuditStorage))
def export_exam_authorizations(self):
"""
Sync any outstanding profiles
"""
if not settings.FEATURES.get("PEARSON_EXAMS_SYNC", False):
return
exam_authorizations = ExamAuthorization.objects.filter(
status=ExamAuthorization.STATUS_PENDING).prefetch_related(
'user__profile', 'course__program')
file_prefix = now_in_utc().strftime(PEARSON_EAD_FILE_PREFIX)
# write the file out locally
# this will be written out to a file like: /tmp/ead-20160405_kjfiamdf.dat
with tempfile.NamedTemporaryFile(
prefix=file_prefix,
encoding=PEARSON_FILE_ENCODING,
suffix=PEARSON_FILE_EXTENSION,
mode='w',
) as tsv:
ead_writer = writers.EADWriter()
valid_auths, _ = ead_writer.write(tsv, exam_authorizations)
# flush data to disk before upload
tsv.flush()
try:
audit.ExamDataAuditor().audit_request_file(tsv.name)
except ImproperlyConfigured:
log.exception('Exam auditing improperly configured')
return
except: # pylint: disable=bare-except
log.exception('Unexpected error auditing EAD file')
return
try:
# upload to SFTP server
upload.upload_tsv(tsv.name)
except ImproperlyConfigured:
log.exception(
'export_exam_authorizations is improperly configured, please review require settings.'
)
except RetryableSFTPException as exc:
log.exception(
'Retryable error during upload of EAD file to Pearson SFTP')
# retry up to 3 times w/ exponential backoff if this was a connection error
self.retry(exc=exc,
countdown=exponential_backoff(self.request.retries))
except: # pylint: disable=bare-except
log.exception('Unexpected exception uploading EAD file')
return
valid_auth_ids = [exam_auth.id for exam_auth in valid_auths]
# update records to reflect the successful upload
if valid_auth_ids:
try:
ExamAuthorization.objects.filter(id__in=valid_auth_ids).update(
status=ExamAuthorization.STATUS_IN_PROGRESS)
except: # pylint: disable=bare-except
log.exception('Unexpected exception updating ExamProfile.status')
def __init__(self, sftp):
self.sftp = sftp
self.auditor = audit.ExamDataAuditor()