def send_grade_to_credentials(self, username, course_run_key, verified, letter_grade, percent_grade):
""" Celery task to notify the Credentials IDA of a grade change via POST. """
logger.info(u'Running task send_grade_to_credentials for username %s and course %s', username, course_run_key)
countdown = 2 ** self.request.retries
course_key = CourseKey.from_string(course_run_key)
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
org=course_key.org,
)
credentials_client.grades.post({
'username': username,
'course_run': str(course_key),
'letter_grade': letter_grade,
'percent_grade': percent_grade,
'verified': verified,
})
logger.info(u'Sent grade for course %s to user %s', course_run_key, username)
except Exception as exc:
logger.exception(u'Failed to send grade for course %s to user %s', course_run_key, username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
def send_grade_to_credentials(self, username, course_run_key, verified,
letter_grade, percent_grade):
""" Celery task to notify the Credentials IDA of a grade change via POST. """
logger.info(
f"Running task send_grade_to_credentials for username {username} and course {course_run_key}"
)
countdown = 2**self.request.retries
course_key = CourseKey.from_string(course_run_key)
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
org=course_key.org,
)
credentials_client.grades.post({
'username': username,
'course_run': str(course_key),
'letter_grade': letter_grade,
'percent_grade': percent_grade,
'verified': verified,
})
logger.info(
f"Sent grade for course {course_run_key} to user {username}")
except Exception as exc: # lint-amnesty, pylint: disable=unused-variable
error_msg = f"Failed to send grade for course {course_run_key} to user {username}."
logger.exception(error_msg)
exception = MaxRetriesExceededError(
f"Failed to send grade to credentials. Reason: {error_msg}")
raise self.retry(exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES)
def backfill_date_for_all_course_runs():
"""
This task will update the course certificate configuration's certificate_available_date
in credentials for all course runs. This is different from the "visable_date" attribute.
This date will always either be the available date that is set in studio for a given course, or it will be None.
This will exclude any course runs that do not have a certificate_available_date or are self paced.
"""
course_run_list = CourseOverview.objects.exclude(self_paced=True).exclude(
certificate_available_date=None)
for index, course_run in enumerate(course_run_list):
logger.info(
f"updating certificate_available_date for course {course_run.id} "
f"with date {course_run.certificate_available_date}")
course_key = str(course_run.id)
course_modes = CourseMode.objects.filter(course_id=course_key)
# There should only ever be one certificate relevant mode per course run
modes = [
mode.slug for mode in course_modes
if mode.slug in CourseMode.CERTIFICATE_RELEVANT_MODES
]
if len(modes) != 1:
logger.exception(
f'Either course {course_key} has no certificate mode or multiple modes. Task failed.'
)
# if there is only one relevant mode, post to credentials
else:
try:
credentials_client = get_credentials_api_client(
User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME), )
api_url = urljoin(f"{get_credentials_api_base_url()}/",
"course_certificates/")
response = credentials_client.post(
api_url,
json={
"course_id":
course_key,
"certificate_type":
modes[0],
"certificate_available_date":
course_run.certificate_available_date.strftime(
'%Y-%m-%dT%H:%M:%SZ'),
"is_active":
True,
})
response.raise_for_status()
logger.info(
f"certificate_available_date updated for course {course_key}"
)
except Exception: # lint-amnesty, pylint: disable=W0703
error_msg = f"Failed to send certificate_available_date for course {course_key}."
logger.exception(error_msg)
if index % 10 == 0:
time.sleep(3)
def clean_certificate_available_date():
"""
This task will clean out the misconfigured certificate available date. When courses Change their
certificates_display_behavior, the certificate_available_date was not updating properly. This is
command is meant to be ran one time to clean up any courses that were not supposed to have
certificate_available_date
"""
course_run_list = CourseOverview.objects.exclude(
self_paced=0,
certificates_display_behavior="end",
certificate_available_date__isnull=False)
for index, course_run in enumerate(course_run_list):
logger.info(
f"removing certificate_available_date for course {course_run.id}")
course_key = str(course_run.id)
course_modes = CourseMode.objects.filter(course_id=course_key)
# There should only ever be one certificate relevant mode per course run
modes = [
mode.slug for mode in course_modes
if mode.slug in CourseMode.CERTIFICATE_RELEVANT_MODES
]
if len(modes) != 1:
logger.exception(
f'Either course {course_key} has no certificate mode or multiple modes. Task failed.'
)
# if there is only one relevant mode, post to credentials
else:
try:
credentials_client = get_credentials_api_client(
User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME), )
credentials_api_base_url = get_credentials_api_base_url()
api_url = urljoin(f"{credentials_api_base_url}/",
"course_certificates/")
response = credentials_client.post(
api_url,
json={
"course_id": course_key,
"certificate_type": modes[0],
"certificate_available_date": None,
"is_active": True,
})
response.raise_for_status()
logger.info(
f"certificate_available_date updated for course {course_key}"
)
except Exception: # lint-amnesty, pylint: disable=W0703
error_msg = f"Failed to send certificate_available_date for course {course_key}."
logger.exception(error_msg)
if index % 10 == 0:
time.sleep(3)
def send_grade_to_credentials(self, username, course_run_key, verified,
letter_grade, percent_grade):
"""
Celery task to notify the Credentials IDA of a grade change via POST.
"""
logger.info(
f"Running task send_grade_to_credentials for username {username} and course {course_run_key}"
)
countdown = 2**self.request.retries
course_key = CourseKey.from_string(course_run_key)
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME))
api_url = urljoin(
f"{get_credentials_api_base_url(org=course_key.org)}/", "grades/")
response = credentials_client.post(api_url,
data={
'username': username,
'course_run': str(course_key),
'letter_grade': letter_grade,
'percent_grade': percent_grade,
'verified': verified,
})
response.raise_for_status()
logger.info(
f"Sent grade for course {course_run_key} to user {username}")
except Exception: # lint-amnesty, pylint: disable=W0703
grade_str = f'(percent: {percent_grade} letter: {letter_grade})'
error_msg = f'Failed to send grade{grade_str} for course {course_run_key} to user {username}.'
logger.exception(error_msg)
exception = MaxRetriesExceededError(
f"Failed to send grade to credentials. Reason: {error_msg}")
raise self.retry(exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES) # pylint: disable=raise-missing-from
def update_credentials_course_certificate_configuration_available_date(
self, course_key, certificate_available_date=None):
"""
This task will update the course certificate configuration's available date. This is different from the
"visable_date" attribute. This date will always either be the available date that is set in studio for
a given course, or it will be None.
Arguments:
course_run_key (str): The course run key to award the certificate for
certificate_available_date (str): A string representation of the datetime for when to make the certificate
available to the user. If not provided, it will be none.
"""
LOGGER.info(
f"Running task update_credentials_course_certificate_configuration_available_date for course {course_key}"
)
course_key = str(course_key)
course_modes = CourseMode.objects.filter(course_id=course_key)
# There should only ever be one certificate relevant mode per course run
modes = [
mode.slug for mode in course_modes
if mode.slug in CourseMode.CERTIFICATE_RELEVANT_MODES
]
if len(modes) != 1:
LOGGER.exception(
f'Either course {course_key} has no certificate mode or multiple modes. Task failed.'
)
return
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME), )
cert_config = {
'course_id': course_key,
'mode': modes[0],
}
post_course_certificate_configuration(
client=credentials_client,
cert_config=cert_config,
certificate_available_date=certificate_available_date)
def award_course_certificate(self, username, course_run_key):
"""
This task is designed to be called whenever a student GeneratedCertificate is updated.
It can be called independently for a username and a course_run, but is invoked on each GeneratedCertificate.save.
"""
LOGGER.info(u'Running task award_course_certificate for username %s', username)
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_course_certificate cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
course_key = CourseKey.from_string(course_run_key)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(u'Task award_course_certificate was called with invalid username %s', username)
# Don't retry for this case - just conclude the task.
return
# Get the cert for the course key and username if it's both passing and available in professional/verified
try:
certificate = GeneratedCertificate.eligible_certificates.get(
user=user.id,
course_id=course_key
)
except GeneratedCertificate.DoesNotExist:
LOGGER.exception(
u'Task award_course_certificate was called without Certificate found for %s to user %s',
course_key,
username
)
return
if certificate.mode in CourseMode.CERTIFICATE_RELEVANT_MODES:
try:
course_overview = CourseOverview.get_from_id(course_key)
except (CourseOverview.DoesNotExist, IOError):
LOGGER.exception(
u'Task award_course_certificate was called without course overview data for course %s',
course_key
)
return
credentials_client = get_credentials_api_client(User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME),
org=course_key.org,
)
# FIXME This may result in visible dates that do not update alongside the Course Overview if that changes
# This is a known limitation of this implementation and was chosen to reduce the amount of replication,
# endpoints, celery tasks, and jenkins jobs that needed to be written for this functionality
visible_date = available_date_for_certificate(course_overview, certificate)
post_course_certificate(credentials_client, username, certificate, visible_date)
LOGGER.info(u'Awarded certificate for course %s to user %s', course_key, username)
except Exception as exc:
LOGGER.exception(u'Failed to determine course certificates to be awarded for user %s', username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
def award_program_certificates(self, username):
"""
This task is designed to be called whenever a student's completion status
changes with respect to one or more courses (primarily, when a course
certificate is awarded).
It will consult with a variety of APIs to determine whether or not the
specified user should be awarded a certificate in one or more programs, and
use the credentials service to create said certificates if so.
This task may also be invoked independently of any course completion status
change - for example, to backpopulate missing program credentials for a
student.
Args:
username (str): The username of the student
Returns:
None
"""
LOGGER.info(u'Running task award_program_certificates for username %s', username)
programs_without_certificates = configuration_helpers.get_value('programs_without_certificates', [])
if programs_without_certificates:
if str(programs_without_certificates[0]).lower() == "all":
# this check will prevent unnecessary logging for partners without program certificates
return
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(u'Task award_program_certificates was called with invalid username %s', username)
# Don't retry for this case - just conclude the task.
return
completed_programs = {}
for site in Site.objects.all():
completed_programs.update(get_completed_programs(site, student))
if not completed_programs:
# No reason to continue beyond this point unless/until this
# task gets updated to support revocation of program certs.
LOGGER.info(u'Task award_program_certificates was called for user %s with no completed programs', username)
return
# Determine which program certificates the user has already been awarded, if any.
existing_program_uuids = get_certified_programs(student)
# we will skip all the programs which have already been awarded and we want to skip the programs
# which are exit in site configuration in 'programs_without_certificates' list.
awarded_and_skipped_program_uuids = list(set(existing_program_uuids + list(programs_without_certificates)))
except Exception as exc:
LOGGER.exception(u'Failed to determine program certificates to be awarded for user %s', username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
# For each completed program for which the student doesn't already have a
# certificate, award one now.
#
# This logic is important, because we will retry the whole task if awarding any particular program cert fails.
#
# N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
new_program_uuids = sorted(list(set(completed_programs.keys()) - set(awarded_and_skipped_program_uuids)))
if new_program_uuids:
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
)
except Exception as exc:
LOGGER.exception('Failed to create a credentials API client to award program certificates')
# Retry because a misconfiguration could be fixed
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
failed_program_certificate_award_attempts = []
for program_uuid in new_program_uuids:
visible_date = completed_programs[program_uuid]
try:
award_program_certificate(credentials_client, username, program_uuid, visible_date)
LOGGER.info(u'Awarded certificate for program %s to user %s', program_uuid, username)
except exceptions.HttpNotFoundError:
LOGGER.exception(
u"""Certificate for program {uuid} could not be found. Unable to award certificate to user
{username}. The program might not be configured.""".format(uuid=program_uuid, username=username)
)
except exceptions.HttpClientError as exc:
# Grab the status code from the client error, because our API
# client handles all 4XX errors the same way. In the future,
# we may want to fork slumber, add 429 handling, and use that
# in edx_rest_api_client.
if exc.response.status_code == 429: # pylint: disable=no-member
rate_limit_countdown = 60
LOGGER.info(
u"""Rate limited. Retrying task to award certificates to user {username} in {countdown}
seconds""".format(username=username, countdown=rate_limit_countdown)
)
# Retry after 60 seconds, when we should be in a new throttling window
raise self.retry(exc=exc, countdown=rate_limit_countdown, max_retries=MAX_RETRIES)
else:
LOGGER.exception(
u"""Unable to award certificate to user {username} for program {uuid}. The program might not be
configured.""".format(username=username, uuid=program_uuid)
)
except Exception: # pylint: disable=broad-except
# keep trying to award other certs, but retry the whole task to fix any missing entries
LOGGER.warning(u'Failed to award certificate for program {uuid} to user {username}.'.format(
uuid=program_uuid, username=username))
failed_program_certificate_award_attempts.append(program_uuid)
if failed_program_certificate_award_attempts:
# N.B. This logic assumes that this task is idempotent
LOGGER.info(u'Retrying task to award failed certificates to user %s', username)
# The error message may change on each reattempt but will never be raised until
# the max number of retries have been exceeded. It is unlikely that this list
# will change by the time it reaches its maximimum number of attempts.
exception = MaxRetriesExceededError(
u"Failed to award certificate for user {} for programs {}".format(
username, failed_program_certificate_award_attempts))
raise self.retry(
exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES)
else:
LOGGER.info(u'User %s is not eligible for any new program certificates', username)
LOGGER.info(u'Successfully completed the task award_program_certificates for username %s', username)
def revoke_program_certificates(self, username, course_key):
"""
This task is designed to be called whenever a student's course certificate is
revoked.
It will consult with a variety of APIs to determine whether or not the
specified user's certificate should be revoked in one or more programs, and
use the credentials service to revoke the said certificates if so.
Args:
username (str): The username of the student
course_key (str|CourseKey): The course identifier
Returns:
None
"""
def _retry_with_custom_exception(username, course_key, reason, countdown):
exception = MaxRetriesExceededError(
f"Failed to revoke program certificate for user {username} for course {course_key}. Reason: {reason}"
)
return self.retry(
exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES
)
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
error_msg = (
"Task revoke_program_certificates cannot be executed when credentials issuance is disabled in API config"
)
LOGGER.warning(error_msg)
raise _retry_with_custom_exception(
username=username,
course_key=course_key,
reason=error_msg,
countdown=countdown
)
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(f"Task revoke_program_certificates was called with invalid username {username}", username)
# Don't retry for this case - just conclude the task.
return
try:
inverted_programs = get_inverted_programs(student)
course_specific_programs = inverted_programs.get(str(course_key))
if not course_specific_programs:
# No reason to continue beyond this point
LOGGER.info(
f"Task revoke_program_certificates was called for user {username} "
f"and course {course_key} with no engaged programs"
)
return
# Determine which program certificates the user has already been awarded, if any.
program_uuids_to_revoke = get_revokable_program_uuids(course_specific_programs, student)
except Exception as exc:
error_msg = (
f"Failed to determine program certificates to be revoked for user {username} "
f"with course {course_key}"
)
LOGGER.exception(error_msg)
raise _retry_with_custom_exception(
username=username,
course_key=course_key,
reason=error_msg,
countdown=countdown
) from exc
if program_uuids_to_revoke:
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
)
except Exception as exc:
error_msg = "Failed to create a credentials API client to revoke program certificates"
LOGGER.exception(error_msg)
# Retry because a misconfiguration could be fixed
raise _retry_with_custom_exception(username, course_key, reason=exc, countdown=countdown) from exc
failed_program_certificate_revoke_attempts = []
for program_uuid in program_uuids_to_revoke:
try:
revoke_program_certificate(credentials_client, username, program_uuid)
LOGGER.info(f"Revoked certificate for program {program_uuid} for user {username}")
except exceptions.HttpNotFoundError:
LOGGER.exception(
f"Certificate for program {program_uuid} could not be found. "
f"Unable to revoke certificate for user {username}"
)
except exceptions.HttpClientError as exc:
# Grab the status code from the client error, because our API
# client handles all 4XX errors the same way. In the future,
# we may want to fork slumber, add 429 handling, and use that
# in edx_rest_api_client.
if exc.response.status_code == 429: # pylint: disable=no-member, no-else-raise
rate_limit_countdown = 60
error_msg = (
"Rate limited. "
f"Retrying task to revoke certificates for user {username} in {rate_limit_countdown} seconds"
)
LOGGER.info(error_msg)
# Retry after 60 seconds, when we should be in a new throttling window
raise _retry_with_custom_exception(
username,
course_key,
reason=error_msg,
countdown=rate_limit_countdown
) from exc
else:
LOGGER.exception(f"Unable to revoke certificate for user {username} for program {program_uuid}.")
except Exception: # pylint: disable=broad-except
# keep trying to revoke other certs, but retry the whole task to fix any missing entries
LOGGER.warning(f"Failed to revoke certificate for program {program_uuid} of user {username}.")
failed_program_certificate_revoke_attempts.append(program_uuid)
if failed_program_certificate_revoke_attempts:
# N.B. This logic assumes that this task is idempotent
LOGGER.info(f"Retrying task to revoke failed certificates to user {username}")
# The error message may change on each reattempt but will never be raised until
# the max number of retries have been exceeded. It is unlikely that this list
# will change by the time it reaches its maximimum number of attempts.
error_msg = (
f"Failed to revoke certificate for user {username} "
f"for programs {failed_program_certificate_revoke_attempts}"
)
raise _retry_with_custom_exception(
username,
course_key,
reason=error_msg,
countdown=countdown
)
else:
LOGGER.info(f"There is no program certificates for user {username} to revoke")
LOGGER.info(f"Successfully completed the task revoke_program_certificates for username {username}")
def award_course_certificate(self, username, course_run_key, certificate_available_date=None):
"""
This task is designed to be called whenever a student GeneratedCertificate is updated.
It can be called independently for a username and a course_run, but is invoked on each GeneratedCertificate.save.
Arguments:
username (str): The user to award the Credentials course cert to
course_run_key (str): The course run key to award the certificate for
certificate_available_date (str): A string representation of the datetime for when to make the certificate
available to the user. If not provided, it will calculate the date.
"""
def _retry_with_custom_exception(username, course_run_key, reason, countdown):
exception = MaxRetriesExceededError(
f"Failed to award course certificate for user {username} for course {course_run_key}. Reason: {reason}"
)
return self.retry(
exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES
)
LOGGER.info(f"Running task award_course_certificate for username {username}")
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
error_msg = (
"Task award_course_certificate cannot be executed when credentials issuance is disabled in API config"
)
LOGGER.warning(error_msg)
raise _retry_with_custom_exception(
username=username,
course_run_key=course_run_key,
reason=error_msg,
countdown=countdown
)
try:
course_key = CourseKey.from_string(course_run_key)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(f"Task award_course_certificate was called with invalid username {username}")
# Don't retry for this case - just conclude the task.
return
# Get the cert for the course key and username if it's both passing and available in professional/verified
try:
certificate = GeneratedCertificate.eligible_certificates.get(
user=user.id,
course_id=course_key
)
except GeneratedCertificate.DoesNotExist:
LOGGER.exception(
"Task award_course_certificate was called without Certificate found "
f"for {course_key} to user {username}"
)
return
if certificate.mode in CourseMode.CERTIFICATE_RELEVANT_MODES:
try:
course_overview = CourseOverview.get_from_id(course_key)
except (CourseOverview.DoesNotExist, IOError):
LOGGER.exception(
f"Task award_course_certificate was called without course overview data for course {course_key}"
)
return
credentials_client = get_credentials_api_client(User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME),
org=course_key.org,
)
# Date is being passed via JSON and is encoded in the EMCA date time string format. The rest of the code
# expects a datetime.
if certificate_available_date:
certificate_available_date = datetime.strptime(certificate_available_date, VISIBLE_DATE_FORMAT)
# Even in the cases where this task is called with a certificate_available_date, we still need to retrieve
# the course overview because it's required to determine if we should use the certificate_available_date or
# the certs modified date
visible_date = available_date_for_certificate(
course_overview,
certificate,
certificate_available_date=certificate_available_date
)
LOGGER.info(
"Task award_course_certificate will award certificate for course "
f"{course_key} with a visible date of {visible_date}"
)
post_course_certificate(credentials_client, username, certificate, visible_date)
LOGGER.info(f"Awarded certificate for course {course_key} to user {username}")
except Exception as exc:
error_msg = f"Failed to determine course certificates to be awarded for user {username}."
LOGGER.exception(error_msg)
raise _retry_with_custom_exception(
username=username,
course_run_key=course_run_key,
reason=error_msg,
countdown=countdown
) from exc
def award_program_certificates(self, username): # lint-amnesty, pylint: disable=too-many-statements
"""
This task is designed to be called whenever a student's completion status
changes with respect to one or more courses (primarily, when a course
certificate is awarded).
It will consult with a variety of APIs to determine whether or not the
specified user should be awarded a certificate in one or more programs, and
use the credentials service to create said certificates if so.
This task may also be invoked independently of any course completion status
change - for example, to backpopulate missing program credentials for a
student.
Args:
username (str): The username of the student
Returns:
None
"""
def _retry_with_custom_exception(username, reason, countdown):
exception = MaxRetriesExceededError(
f"Failed to award program certificate for user {username}. Reason: {reason}"
)
return self.retry(
exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES
)
LOGGER.info(f"Running task award_program_certificates for username {username}")
programs_without_certificates = configuration_helpers.get_value('programs_without_certificates', [])
if programs_without_certificates:
if str(programs_without_certificates[0]).lower() == "all":
# this check will prevent unnecessary logging for partners without program certificates
return
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
error_msg = (
"Task award_program_certificates cannot be executed when credentials issuance is disabled in API config"
)
LOGGER.warning(error_msg)
raise _retry_with_custom_exception(username=username, reason=error_msg, countdown=countdown)
try:
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(f"Task award_program_certificates was called with invalid username {username}")
# Don't retry for this case - just conclude the task.
return
completed_programs = {}
for site in Site.objects.all():
completed_programs.update(get_completed_programs(site, student))
if not completed_programs:
# No reason to continue beyond this point unless/until this
# task gets updated to support revocation of program certs.
LOGGER.info(f"Task award_program_certificates was called for user {username} with no completed programs")
return
# Determine which program certificates the user has already been awarded, if any.
existing_program_uuids = get_certified_programs(student)
# we will skip all the programs which have already been awarded and we want to skip the programs
# which are exit in site configuration in 'programs_without_certificates' list.
awarded_and_skipped_program_uuids = list(set(existing_program_uuids + list(programs_without_certificates)))
except Exception as exc:
error_msg = f"Failed to determine program certificates to be awarded for user {username}. {exc}"
LOGGER.exception(error_msg)
raise _retry_with_custom_exception(username=username, reason=error_msg, countdown=countdown) from exc
# For each completed program for which the student doesn't already have a
# certificate, award one now.
#
# This logic is important, because we will retry the whole task if awarding any particular program cert fails.
#
# N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
new_program_uuids = sorted(list(set(completed_programs.keys()) - set(awarded_and_skipped_program_uuids)))
if new_program_uuids:
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
)
except Exception as exc:
error_msg = "Failed to create a credentials API client to award program certificates"
LOGGER.exception(error_msg)
# Retry because a misconfiguration could be fixed
raise _retry_with_custom_exception(username=username, reason=error_msg, countdown=countdown) from exc
failed_program_certificate_award_attempts = []
for program_uuid in new_program_uuids:
visible_date = completed_programs[program_uuid]
try:
LOGGER.info(f"Visible date for user {username} : program {program_uuid} is {visible_date}")
award_program_certificate(credentials_client, username, program_uuid, visible_date)
LOGGER.info(f"Awarded certificate for program {program_uuid} to user {username}")
except exceptions.HttpNotFoundError:
LOGGER.exception(
f"Certificate for program {program_uuid} could not be found. " +
f"Unable to award certificate to user {username}. The program might not be configured."
)
except exceptions.HttpClientError as exc:
# Grab the status code from the client error, because our API
# client handles all 4XX errors the same way. In the future,
# we may want to fork slumber, add 429 handling, and use that
# in edx_rest_api_client.
if exc.response.status_code == 429: # lint-amnesty, pylint: disable=no-else-raise, no-member
rate_limit_countdown = 60
error_msg = (
f"Rate limited. "
f"Retrying task to award certificates to user {username} in {rate_limit_countdown} seconds"
)
LOGGER.info(error_msg)
# Retry after 60 seconds, when we should be in a new throttling window
raise _retry_with_custom_exception(
username=username,
reason=error_msg,
countdown=rate_limit_countdown
) from exc
else:
LOGGER.exception(
f"Unable to award certificate to user {username} for program {program_uuid}. "
"The program might not be configured."
)
except Exception as exc: # pylint: disable=broad-except
# keep trying to award other certs, but retry the whole task to fix any missing entries
LOGGER.exception(f"Failed to award certificate for program {program_uuid} to user {username}.")
failed_program_certificate_award_attempts.append(program_uuid)
if failed_program_certificate_award_attempts:
# N.B. This logic assumes that this task is idempotent
LOGGER.info(f"Retrying task to award failed certificates to user {username}")
# The error message may change on each reattempt but will never be raised until
# the max number of retries have been exceeded. It is unlikely that this list
# will change by the time it reaches its maximimum number of attempts.
error_msg = (
f"Failed to award certificate for user {username} "
f"for programs {failed_program_certificate_award_attempts}"
)
raise _retry_with_custom_exception(username=username, reason=error_msg, countdown=countdown)
else:
LOGGER.info(f"User {username} is not eligible for any new program certificates")
LOGGER.info(f"Successfully completed the task award_program_certificates for username {username}")
def award_program_certificates(self, username):
"""
This task is designed to be called whenever a student's completion status
changes with respect to one or more courses (primarily, when a course
certificate is awarded).
It will consult with a variety of APIs to determine whether or not the
specified user should be awarded a certificate in one or more programs, and
use the credentials service to create said certificates if so.
This task may also be invoked independently of any course completion status
change - for example, to backpopulate missing program credentials for a
student.
Args:
username (str): The username of the student
Returns:
None
"""
LOGGER.info('Running task award_program_certificates for username %s',
username)
countdown = 2**self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(
'Task award_program_certificates was called with invalid username %s',
username)
# Don't retry for this case - just conclude the task.
return
program_uuids = []
for site in Site.objects.all():
program_uuids.extend(get_completed_programs(site, student))
if not program_uuids:
# No reason to continue beyond this point unless/until this
# task gets updated to support revocation of program certs.
LOGGER.info(
'Task award_program_certificates was called for user %s with no completed programs',
username)
return
# Determine which program certificates the user has already been awarded, if any.
existing_program_uuids = get_certified_programs(student)
except Exception as exc:
LOGGER.exception(
'Failed to determine program certificates to be awarded for user %s',
username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
# For each completed program for which the student doesn't already have a
# certificate, award one now.
#
# This logic is important, because we will retry the whole task if awarding any particular program cert fails.
#
# N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
new_program_uuids = sorted(
list(set(program_uuids) - set(existing_program_uuids)))
if new_program_uuids:
try:
credentials_client = get_credentials_api_client(
User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME), )
except Exception as exc:
LOGGER.exception(
'Failed to create a credentials API client to award program certificates'
)
# Retry because a misconfiguration could be fixed
raise self.retry(exc=exc,
countdown=countdown,
max_retries=MAX_RETRIES)
failed_program_certificate_award_attempts = []
for program_uuid in new_program_uuids:
try:
award_program_certificate(credentials_client, username,
program_uuid)
LOGGER.info('Awarded certificate for program %s to user %s',
program_uuid, username)
except exceptions.HttpNotFoundError:
LOGGER.exception(
"""Certificate for program {uuid} could not be found. Unable to award certificate to user
{username}. The program might not be configured.""".format(
uuid=program_uuid, username=username))
except exceptions.HttpClientError as exc:
# Grab the status code from the client error, because our API
# client handles all 4XX errors the same way. In the future,
# we may want to fork slumber, add 429 handling, and use that
# in edx_rest_api_client.
if exc.response.status_code == 429: # pylint: disable=no-member
rate_limit_countdown = 60
LOGGER.info(
"""Rate limited. Retrying task to award certificates to user {username} in {countdown}
seconds""".format(username=username,
countdown=rate_limit_countdown))
# Retry after 60 seconds, when we should be in a new throttling window
raise self.retry(exc=exc,
countdown=rate_limit_countdown,
max_retries=MAX_RETRIES)
else:
LOGGER.exception(
"""Unable to award certificate to user {username} for program {uuid}. The program might not be
configured.""".format(username=username,
uuid=program_uuid))
except Exception: # pylint: disable=broad-except
# keep trying to award other certs, but retry the whole task to fix any missing entries
LOGGER.warning(
'Failed to award certificate for program {uuid} to user {username}.'
.format(uuid=program_uuid, username=username))
failed_program_certificate_award_attempts.append(program_uuid)
if failed_program_certificate_award_attempts:
# N.B. This logic assumes that this task is idempotent
LOGGER.info(
'Retrying task to award failed certificates to user %s',
username)
# The error message may change on each reattempt but will never be raised until
# the max number of retries have been exceeded. It is unlikely that this list
# will change by the time it reaches its maximimum number of attempts.
exception = MaxRetriesExceededError(
"Failed to award certificate for user {} for programs {}".
format(username, failed_program_certificate_award_attempts))
raise self.retry(exc=exception,
countdown=countdown,
max_retries=MAX_RETRIES)
else:
LOGGER.info('User %s is not eligible for any new program certificates',
username)
LOGGER.info(
'Successfully completed the task award_program_certificates for username %s',
username)
def award_course_certificate(self, username, course_run_key):
"""
This task is designed to be called whenever a student GeneratedCertificate is updated.
It can be called independently for a username and a course_run, but is invoked on each GeneratedCertificate.save.
"""
LOGGER.info('Running task award_course_certificate for username %s',
username)
countdown = 2**self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_course_certificate cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
course_key = CourseKey.from_string(course_run_key)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(
'Task award_course_certificate was called with invalid username %s',
username)
# Don't retry for this case - just conclude the task.
return
# Get the cert for the course key and username if it's both passing and available in professional/verified
try:
certificate = GeneratedCertificate.eligible_certificates.get(
user=user.id, course_id=course_key)
except GeneratedCertificate.DoesNotExist:
LOGGER.exception(
'Task award_course_certificate was called without Certificate found for %s to user %s',
course_key, username)
return
if certificate.mode in CourseMode.CREDIT_ELIGIBLE_MODES + CourseMode.CREDIT_MODES:
try:
course_overview = CourseOverview.get_from_id(course_key)
except (CourseOverview.DoesNotExist, IOError):
LOGGER.exception(
'Task award_course_certificate was called without course overview data for course %s',
course_key)
return
credentials_client = get_credentials_api_client(
User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME),
org=course_key.org,
)
# FIXME This may result in visible dates that do not update alongside the Course Overview if that changes
# This is a known limitation of this implementation and was chosen to reduce the amount of replication,
# endpoints, celery tasks, and jenkins jobs that needed to be written for this functionality
visible_date = available_date_for_certificate(
course_overview, certificate)
post_course_certificate(credentials_client, username, certificate,
visible_date)
LOGGER.info('Awarded certificate for course %s to user %s',
course_key, username)
except Exception as exc:
LOGGER.exception(
'Failed to determine course certificates to be awarded for user %s',
username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
def award_program_certificates(self, username):
"""
This task is designed to be called whenever a student's completion status
changes with respect to one or more courses (primarily, when a course
certificate is awarded).
It will consult with a variety of APIs to determine whether or not the
specified user should be awarded a certificate in one or more programs, and
use the credentials service to create said certificates if so.
This task may also be invoked independently of any course completion status
change - for example, to backpopulate missing program credentials for a
student.
Args:
username (str): The username of the student
Returns:
None
"""
LOGGER.info('Running task award_program_certificates for username %s',
username)
countdown = 2**self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception(
'Task award_program_certificates was called with invalid username %s',
username)
# Don't retry for this case - just conclude the task.
return
program_uuids = []
for site in Site.objects.all():
program_uuids.extend(get_completed_programs(site, student))
if not program_uuids:
# No reason to continue beyond this point unless/until this
# task gets updated to support revocation of program certs.
LOGGER.info(
'Task award_program_certificates was called for user %s with no completed programs',
username)
return
# Determine which program certificates the user has already been awarded, if any.
existing_program_uuids = get_certified_programs(student)
except Exception as exc: # pylint: disable=broad-except
LOGGER.exception(
'Failed to determine program certificates to be awarded for user %s',
username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
# For each completed program for which the student doesn't already have a
# certificate, award one now.
#
# This logic is important, because we will retry the whole task if awarding any particular program cert fails.
#
# N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
new_program_uuids = sorted(
list(set(program_uuids) - set(existing_program_uuids)))
if new_program_uuids:
try:
credentials_client = get_credentials_api_client(
User.objects.get(
username=settings.CREDENTIALS_SERVICE_USERNAME), )
except Exception as exc: # pylint: disable=broad-except
LOGGER.exception(
'Failed to create a credentials API client to award program certificates'
)
# Retry because a misconfiguration could be fixed
raise self.retry(exc=exc,
countdown=countdown,
max_retries=MAX_RETRIES)
retry = False
for program_uuid in new_program_uuids:
try:
award_program_certificate(credentials_client, username,
program_uuid)
LOGGER.info('Awarded certificate for program %s to user %s',
program_uuid, username)
except exceptions.HttpClientError:
LOGGER.exception(
'Certificate for program %s not configured, unable to award certificate to %s',
program_uuid, username)
except Exception: # pylint: disable=broad-except
# keep trying to award other certs, but retry the whole task to fix any missing entries
LOGGER.warning(
'Failed to award certificate for program {uuid} to user {username}.'
.format(uuid=program_uuid, username=username))
retry = True
if retry:
# N.B. This logic assumes that this task is idempotent
LOGGER.info(
'Retrying task to award failed certificates to user %s',
username)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
else:
LOGGER.info('User %s is not eligible for any new program certificates',
username)
LOGGER.info(
'Successfully completed the task award_program_certificates for username %s',
username)
def award_program_certificates(self, username):
"""
This task is designed to be called whenever a student's completion status
changes with respect to one or more courses (primarily, when a course
certificate is awarded).
It will consult with a variety of APIs to determine whether or not the
specified user should be awarded a certificate in one or more programs, and
use the credentials service to create said certificates if so.
This task may also be invoked independently of any course completion status
change - for example, to backpopulate missing program credentials for a
student.
Args:
username (str): The username of the student
Returns:
None
"""
LOGGER.info('Running task award_program_certificates for username %s', username)
countdown = 2 ** self.request.retries
# If the credentials config model is disabled for this
# feature, it may indicate a condition where processing of such tasks
# has been temporarily disabled. Since this is a recoverable situation,
# mark this task for retry instead of failing it altogether.
if not CredentialsApiConfig.current().is_learner_issuance_enabled:
LOGGER.warning(
'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
try:
try:
student = User.objects.get(username=username)
except User.DoesNotExist:
LOGGER.exception('Task award_program_certificates was called with invalid username %s', username)
# Don't retry for this case - just conclude the task.
return
program_uuids = []
for site in Site.objects.all():
program_uuids.extend(get_completed_programs(site, student))
if not program_uuids:
# No reason to continue beyond this point unless/until this
# task gets updated to support revocation of program certs.
LOGGER.info('Task award_program_certificates was called for user %s with no completed programs', username)
return
# Determine which program certificates the user has already been awarded, if any.
existing_program_uuids = get_certified_programs(student)
except Exception as exc: # pylint: disable=broad-except
LOGGER.exception('Failed to determine program certificates to be awarded for user %s', username)
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
# For each completed program for which the student doesn't already have a
# certificate, award one now.
#
# This logic is important, because we will retry the whole task if awarding any particular program cert fails.
#
# N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
new_program_uuids = sorted(list(set(program_uuids) - set(existing_program_uuids)))
if new_program_uuids:
try:
credentials_client = get_credentials_api_client(
User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME),
)
except Exception as exc: # pylint: disable=broad-except
LOGGER.exception('Failed to create a credentials API client to award program certificates')
# Retry because a misconfiguration could be fixed
raise self.retry(exc=exc, countdown=countdown, max_retries=MAX_RETRIES)
retry = False
for program_uuid in new_program_uuids:
try:
award_program_certificate(credentials_client, username, program_uuid)
LOGGER.info('Awarded certificate for program %s to user %s', program_uuid, username)
except exceptions.HttpNotFoundError:
LOGGER.exception(
'Certificate for program %s not configured, unable to award certificate to %s',
program_uuid, username
)
except Exception: # pylint: disable=broad-except
# keep trying to award other certs, but retry the whole task to fix any missing entries
LOGGER.warning('Failed to award certificate for program {uuid} to user {username}.'.format(
uuid=program_uuid, username=username))
retry = True
if retry:
# N.B. This logic assumes that this task is idempotent
LOGGER.info('Retrying task to award failed certificates to user %s', username)
raise self.retry(countdown=countdown, max_retries=MAX_RETRIES)
else:
LOGGER.info('User %s is not eligible for any new program certificates', username)
LOGGER.info('Successfully completed the task award_program_certificates for username %s', username)
