def _create_or_rerun_course(request):
"""
To be called by requests that create a new destination course (i.e., create_new_course and rerun_course)
Returns the destination course_key and overriding fields for the new course.
Raises InvalidLocationError and InvalidKeyError
"""
if not auth.has_access(request.user, CourseCreatorRole()):
raise PermissionDenied()
try:
org = request.json.get('org')
number = request.json.get('number')
display_name = request.json.get('display_name')
run = request.json.get('run')
# allow/disable unicode characters in course_id according to settings
if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
if _has_non_ascii_characters(org) or _has_non_ascii_characters(number) or _has_non_ascii_characters(run):
return JsonResponse(
{'error': _('Special characters not allowed in organization, course number, and course run.')},
status=400
)
course_key = SlashSeparatedCourseKey(org, number, run)
fields = {'display_name': display_name} if display_name is not None else {}
if 'source_course_key' in request.json:
return _rerun_course(request, course_key, fields)
else:
return _create_new_course(request, course_key, fields)
except InvalidLocationError:
return JsonResponse({
'ErrMsg': _(
'There is already a course defined with the same '
'organization, course number, and course run. Please '
'change either organization or course number to be unique.'
),
'OrgErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
'CourseErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
})
except InvalidKeyError as error:
return JsonResponse({
"ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=error.message)}
)
def create_new_course(request):
"""
Create a new course.
Returns the URL for the course overview page.
"""
if not auth.has_access(request.user, CourseCreatorRole()):
raise PermissionDenied()
org = request.json.get('org')
number = request.json.get('number')
display_name = request.json.get('display_name')
run = request.json.get('run')
# allow/disable unicode characters in course_id according to settings
if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
if _has_non_ascii_characters(org) or _has_non_ascii_characters(number) or _has_non_ascii_characters(run):
return JsonResponse(
{'error': _('Special characters not allowed in organization, course number, and course run.')},
status=400
)
try:
course_key = SlashSeparatedCourseKey(org, number, run)
# instantiate the CourseDescriptor and then persist it
# note: no system to pass
if display_name is None:
metadata = {}
else:
metadata = {'display_name': display_name}
# Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for
# existing xml courses this cannot be changed in CourseDescriptor.
# # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile
# w/ xmodule.course_module.CourseDescriptor.__init__
wiki_slug = u"{0}.{1}.{2}".format(course_key.org, course_key.course, course_key.run)
definition_data = {'wiki_slug': wiki_slug}
# Create the course then fetch it from the modulestore
# Check if role permissions group for a course named like this already exists
# Important because role groups are case insensitive
if CourseRole.course_group_already_exists(course_key):
raise InvalidLocationError()
fields = {}
fields.update(definition_data)
fields.update(metadata)
# Creating the course raises InvalidLocationError if an existing course with this org/name is found
new_course = modulestore('direct').create_course(
course_key.org,
course_key.offering,
fields=fields,
)
# can't use auth.add_users here b/c it requires request.user to already have Instructor perms in this course
# however, we can assume that b/c this user had authority to create the course, the user can add themselves
CourseInstructorRole(new_course.id).add_users(request.user)
auth.add_users(request.user, CourseStaffRole(new_course.id), request.user)
# seed the forums
seed_permissions_roles(new_course.id)
# auto-enroll the course creator in the course so that "View Live" will
# work.
CourseEnrollment.enroll(request.user, new_course.id)
_users_assign_default_role(new_course.id)
return JsonResponse({
'url': reverse_course_url('course_handler', new_course.id)
})
except InvalidLocationError:
return JsonResponse({
'ErrMsg': _(
'There is already a course defined with the same '
'organization, course number, and course run. Please '
'change either organization or course number to be unique.'
),
'OrgErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
'CourseErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
})
except InvalidKeyError as error:
return JsonResponse({
"ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=error.message)}
)
def create_new_course(request):
"""
Create a new course.
Returns the URL for the course overview page.
"""
if not auth.has_access(request.user, CourseCreatorRole()):
raise PermissionDenied()
org = request.json.get('org')
number = request.json.get('number')
display_name = request.json.get('display_name')
run = request.json.get('run')
# allow/disable unicode characters in course_id according to settings
if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
if _has_non_ascii_characters(org) or _has_non_ascii_characters(number) or _has_non_ascii_characters(run):
return JsonResponse(
{'error': _('Special characters not allowed in organization, course number, and course run.')},
status=400
)
try:
dest_location = Location(u'i4x', org, number, u'course', run)
except InvalidLocationError as error:
return JsonResponse({
"ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(
name=display_name, err=error.message)})
# see if the course already exists
existing_course = None
try:
existing_course = modulestore('direct').get_item(dest_location)
except ItemNotFoundError:
pass
if existing_course is not None:
return JsonResponse({
'ErrMsg': _(
'There is already a course defined with the same '
'organization, course number, and course run. Please '
'change either organization or course number to be '
'unique.'
),
'OrgErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'
),
'CourseErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'
),
})
# dhm: this query breaks the abstraction, but I'll fix it when I do my suspended refactoring of this
# file for new locators. get_items should accept a query rather than requiring it be a legal location
course_search_location = bson.son.SON({
'_id.tag': 'i4x',
# cannot pass regex to Location constructor; thus this hack
# pylint: disable=E1101
'_id.org': re.compile(u'^{}$'.format(dest_location.org), re.IGNORECASE | re.UNICODE),
# pylint: disable=E1101
'_id.course': re.compile(u'^{}$'.format(dest_location.course), re.IGNORECASE | re.UNICODE),
'_id.category': 'course',
})
courses = modulestore().collection.find(course_search_location, fields=('_id'))
if courses.count() > 0:
return JsonResponse({
'ErrMsg': _(
'There is already a course defined with the same '
'organization and course number. Please '
'change at least one field to be unique.'),
'OrgErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
'CourseErrMsg': _(
'Please change either the organization or '
'course number so that it is unique.'),
})
# instantiate the CourseDescriptor and then persist it
# note: no system to pass
if display_name is None:
metadata = {}
else:
metadata = {'display_name': display_name}
# Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for existing xml courses this
# cannot be changed in CourseDescriptor.
wiki_slug = u"{0}.{1}.{2}".format(dest_location.org, dest_location.course, dest_location.name)
definition_data = {'wiki_slug': wiki_slug}
modulestore('direct').create_and_save_xmodule(
dest_location,
definition_data=definition_data,
metadata=metadata
)
new_course = modulestore('direct').get_item(dest_location)
# clone a default 'about' overview module as well
dest_about_location = dest_location.replace(
category='about',
name='overview'
)
overview_template = AboutDescriptor.get_template('overview.yaml')
modulestore('direct').create_and_save_xmodule(
dest_about_location,
system=new_course.system,
definition_data=overview_template.get('data')
)
new_location = loc_mapper().translate_location(new_course.location.course_id, new_course.location, False, True)
# can't use auth.add_users here b/c it requires request.user to already have Instructor perms in this course
# however, we can assume that b/c this user had authority to create the course, the user can add themselves
CourseInstructorRole(new_location).add_users(request.user)
auth.add_users(request.user, CourseStaffRole(new_location), request.user)
# seed the forums
seed_permissions_roles(new_course.location.course_id)
# auto-enroll the course creator in the course so that "View Live" will
# work.
CourseEnrollment.enroll(request.user, new_course.location.course_id)
_users_assign_default_role(new_course.location)
return JsonResponse({'url': new_location.url_reverse("course/", "")})
def _send_course_email(entry_id, email_id, to_list, global_email_context, subtask_status):
"""
Performs the email sending task.
Sends an email to a list of recipients.
Inputs are:
* `entry_id`: id of the InstructorTask object to which progress should be recorded.
* `email_id`: id of the CourseEmail model that is to be emailed.
* `to_list`: list of recipients. Each is represented as a dict with the following keys:
- 'profile__name': full name of User.
- 'email': email address of User.
- 'pk': primary key of User model.
* `global_email_context`: dict containing values that are unique for this email but the same
for all recipients of this email. This dict is to be used to fill in slots in email
template. It does not include 'name' and 'email', which will be provided by the to_list.
* `subtask_status` : object of class SubtaskStatus representing current status.
Sends to all addresses contained in to_list that are not also in the Optout table.
Emails are sent multi-part, in both plain text and html.
Returns a tuple of two values:
* First value is a SubtaskStatus object which represents current progress at the end of this call.
* Second value is an exception returned by the innards of the method, indicating a fatal error.
In this case, the number of recipients that were not sent have already been added to the
'failed' count above.
"""
# Get information from current task's request:
parent_task_id = InstructorTask.objects.get(pk=entry_id).task_id
task_id = subtask_status.task_id
total_recipients = len(to_list)
recipient_num = 0
total_recipients_successful = 0
total_recipients_failed = 0
recipients_info = Counter()
log.info(
"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, TotalRecipients: %s",
parent_task_id,
task_id,
email_id,
total_recipients
)
try:
course_email = CourseEmail.objects.get(id=email_id)
except CourseEmail.DoesNotExist as exc:
log.exception(
"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Could not find email to send.",
parent_task_id,
task_id,
email_id
)
raise
# Exclude optouts (if not a retry):
# Note that we don't have to do the optout logic at all if this is a retry,
# because we have presumably already performed the optout logic on the first
# attempt. Anyone on the to_list on a retry has already passed the filter
# that existed at that time, and we don't need to keep checking for changes
# in the Optout list.
if subtask_status.get_retry_count() == 0:
to_list, num_optout = _filter_optouts_from_recipients(to_list, course_email.course_id)
subtask_status.increment(skipped=num_optout)
course_title = global_email_context['course_title']
course_language = global_email_context['course_language']
# use the email from address in the CourseEmail, if it is present, otherwise compute it
from_addr = course_email.from_addr if course_email.from_addr else \
_get_source_address(course_email.course_id, course_title, course_language)
# use the CourseEmailTemplate that was associated with the CourseEmail
course_email_template = course_email.get_template()
try:
connection = get_connection()
connection.open()
# Define context values to use in all course emails:
email_context = {'name': '', 'email': ''}
email_context.update(global_email_context)
while to_list:
# Update context with user-specific values from the user at the end of the list.
# At the end of processing this user, they will be popped off of the to_list.
# That way, the to_list will always contain the recipients remaining to be emailed.
# This is convenient for retries, which will need to send to those who haven't
# yet been emailed, but not send to those who have already been sent to.
recipient_num += 1
current_recipient = to_list[-1]
email = current_recipient['email']
if _has_non_ascii_characters(email):
to_list.pop()
total_recipients_failed += 1
log.info(
"BulkEmail ==> Email address %s contains non-ascii characters. Skipping sending "
"email to %s, EmailId: %s ",
email,
current_recipient['profile__name'],
email_id
)
subtask_status.increment(failed=1)
continue
email_context['email'] = email
email_context['name'] = current_recipient['profile__name']
email_context['user_id'] = current_recipient['pk']
email_context['course_id'] = course_email.course_id
# Construct message content using templates and context:
plaintext_msg = course_email_template.render_plaintext(course_email.text_message, email_context)
html_msg = course_email_template.render_htmltext(course_email.html_message, email_context)
# Create email:
email_msg = EmailMultiAlternatives(
course_email.subject,
plaintext_msg,
from_addr,
[email],
connection=connection
)
email_msg.attach_alternative(html_msg, 'text/html')
# Throttle if we have gotten the rate limiter. This is not very high-tech,
# but if a task has been retried for rate-limiting reasons, then we sleep
# for a period of time between all emails within this task. Choice of
# the value depends on the number of workers that might be sending email in
# parallel, and what the SES throttle rate is.
if subtask_status.retried_nomax > 0:
sleep(settings.BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS)
try:
log.info(
"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Recipient num: %s/%s, \
Recipient name: %s, Email address: %s",
parent_task_id,
task_id,
email_id,
recipient_num,
total_recipients,
current_recipient['profile__name'],
email
)
with dog_stats_api.timer('course_email.single_send.time.overall', tags=[_statsd_tag(course_title)]):
connection.send_messages([email_msg])
except SMTPDataError as exc:
# According to SMTP spec, we'll retry error codes in the 4xx range. 5xx range indicates hard failure.
total_recipients_failed += 1
log.error(
"BulkEmail ==> Status: Failed(SMTPDataError), Task: %s, SubTask: %s, EmailId: %s, \
Recipient num: %s/%s, Email address: %s",
parent_task_id,
task_id,
email_id,
recipient_num,
total_recipients,
email
)
if exc.smtp_code >= 400 and exc.smtp_code < 500:
# This will cause the outer handler to catch the exception and retry the entire task.
raise exc
else:
# This will fall through and not retry the message.
log.warning(
'BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Recipient num: %s/%s, \
Email not delivered to %s due to error %s',
parent_task_id,
task_id,
email_id,
recipient_num,
total_recipients,
email,
exc.smtp_error
)
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
subtask_status.increment(failed=1)
except SINGLE_EMAIL_FAILURE_ERRORS as exc:
# This will fall through and not retry the message.
total_recipients_failed += 1
log.error(
"BulkEmail ==> Status: Failed(SINGLE_EMAIL_FAILURE_ERRORS), Task: %s, SubTask: %s, \
EmailId: %s, Recipient num: %s/%s, Email address: %s, Exception: %s",
parent_task_id,
task_id,
email_id,
recipient_num,
total_recipients,
email,
exc
)
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
subtask_status.increment(failed=1)
else:
total_recipients_successful += 1
log.info(
"BulkEmail ==> Status: Success, Task: %s, SubTask: %s, EmailId: %s, \
Recipient num: %s/%s, Email address: %s,",
parent_task_id,
task_id,
email_id,
recipient_num,
total_recipients,
email
)
dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
if settings.BULK_EMAIL_LOG_SENT_EMAILS:
log.info('Email with id %s sent to %s', email_id, email)
else:
log.debug('Email with id %s sent to %s', email_id, email)
subtask_status.increment(succeeded=1)
# Pop the user that was emailed off the end of the list only once they have
# successfully been processed. (That way, if there were a failure that
# needed to be retried, the user is still on the list.)
recipients_info[email] += 1
to_list.pop()
log.info(
"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Total Successful Recipients: %s/%s, \
Failed Recipients: %s/%s",
parent_task_id,
task_id,
email_id,
total_recipients_successful,
total_recipients,
total_recipients_failed,
total_recipients
)
duplicate_recipients = ["{0} ({1})".format(email, repetition)
for email, repetition in recipients_info.most_common() if repetition > 1]
if duplicate_recipients:
log.info(
"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Total Duplicate Recipients [%s]: [%s]",
parent_task_id,
task_id,
email_id,
len(duplicate_recipients),
', '.join(duplicate_recipients)
)
except INFINITE_RETRY_ERRORS as exc:
dog_stats_api.increment('course_email.infinite_retry', tags=[_statsd_tag(course_title)])
# Increment the "retried_nomax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_nomax=1, state=RETRY)
return _submit_for_retry(
entry_id, email_id, to_list, global_email_context, exc, subtask_status, skip_retry_max=True
)
except LIMITED_RETRY_ERRORS as exc:
# Errors caught here cause the email to be retried. The entire task is actually retried
# without popping the current recipient off of the existing list.
# Errors caught are those that indicate a temporary condition that might succeed on retry.
dog_stats_api.increment('course_email.limited_retry', tags=[_statsd_tag(course_title)])
# Increment the "retried_withmax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_withmax=1, state=RETRY)
return _submit_for_retry(
entry_id, email_id, to_list, global_email_context, exc, subtask_status, skip_retry_max=False
)
except BULK_EMAIL_FAILURE_ERRORS as exc:
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
num_pending = len(to_list)
log.exception(('Task %s: email with id %d caused send_course_email task to fail '
'with "fatal" exception. %d emails unsent.'),
task_id, email_id, num_pending)
# Update counters with progress to date, counting unsent emails as failures,
# and set the state to FAILURE:
subtask_status.increment(failed=num_pending, state=FAILURE)
return subtask_status, exc
except Exception as exc: # pylint: disable=broad-except
# Errors caught here cause the email to be retried. The entire task is actually retried
# without popping the current recipient off of the existing list.
# These are unexpected errors. Since they might be due to a temporary condition that might
# succeed on retry, we give them a retry.
dog_stats_api.increment('course_email.limited_retry', tags=[_statsd_tag(course_title)])
log.exception(('Task %s: email with id %d caused send_course_email task to fail '
'with unexpected exception. Generating retry.'),
task_id, email_id)
# Increment the "retried_withmax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_withmax=1, state=RETRY)
return _submit_for_retry(
entry_id, email_id, to_list, global_email_context, exc, subtask_status, skip_retry_max=False
)
else:
# All went well. Update counters with progress to date,
# and set the state to SUCCESS:
subtask_status.increment(state=SUCCESS)
# Successful completion is marked by an exception value of None.
return subtask_status, None
finally:
# Clean up at the end.
connection.close()
def create_new_course(request):
"""
Create a new course.
Returns the URL for the course overview page.
"""
if not auth.has_access(request.user, CourseCreatorRole()):
raise PermissionDenied()
org = request.json.get('org')
number = request.json.get('number')
display_name = request.json.get('display_name')
run = request.json.get('run')
# allow/disable unicode characters in course_id according to settings
if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
if _has_non_ascii_characters(org) or _has_non_ascii_characters(
number) or _has_non_ascii_characters(run):
return JsonResponse(
{
'error':
_('Special characters not allowed in organization, course number, and course run.'
)
},
status=400)
try:
course_key = SlashSeparatedCourseKey(org, number, run)
# instantiate the CourseDescriptor and then persist it
# note: no system to pass
if display_name is None:
metadata = {}
else:
metadata = {'display_name': display_name}
# Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for
# existing xml courses this cannot be changed in CourseDescriptor.
# # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile
# w/ xmodule.course_module.CourseDescriptor.__init__
wiki_slug = u"{0}.{1}.{2}".format(course_key.org, course_key.course,
course_key.run)
definition_data = {'wiki_slug': wiki_slug}
# Create the course then fetch it from the modulestore
# Check if role permissions group for a course named like this already exists
# Important because role groups are case insensitive
if CourseRole.course_group_already_exists(course_key):
raise InvalidLocationError()
fields = {}
fields.update(definition_data)
fields.update(metadata)
# Creating the course raises InvalidLocationError if an existing course with this org/name is found
new_course = modulestore('direct').create_course(
course_key.org,
course_key.offering,
fields=fields,
)
# can't use auth.add_users here b/c it requires request.user to already have Instructor perms in this course
# however, we can assume that b/c this user had authority to create the course, the user can add themselves
CourseInstructorRole(new_course.id).add_users(request.user)
auth.add_users(request.user, CourseStaffRole(new_course.id),
request.user)
# seed the forums
seed_permissions_roles(new_course.id)
# auto-enroll the course creator in the course so that "View Live" will
# work.
CourseEnrollment.enroll(request.user, new_course.id)
_users_assign_default_role(new_course.id)
return JsonResponse(
{'url': reverse_course_url('course_handler', new_course.id)})
except InvalidLocationError:
return JsonResponse({
'ErrMsg':
_('There is already a course defined with the same '
'organization, course number, and course run. Please '
'change either organization or course number to be unique.'),
'OrgErrMsg':
_('Please change either the organization or '
'course number so that it is unique.'),
'CourseErrMsg':
_('Please change either the organization or '
'course number so that it is unique.'),
})
except InvalidKeyError as error:
return JsonResponse({
"ErrMsg":
_("Unable to create course '{name}'.\n\n{err}").format(
name=display_name, err=error.message)
})
def _send_course_email(entry_id, email_id, to_list, global_email_context,
subtask_status):
"""
Performs the email sending task.
Sends an email to a list of recipients.
Inputs are:
* `entry_id`: id of the InstructorTask object to which progress should be recorded.
* `email_id`: id of the CourseEmail model that is to be emailed.
* `to_list`: list of recipients. Each is represented as a dict with the following keys:
- 'profile__name': full name of User.
- 'email': email address of User.
- 'pk': primary key of User model.
* `global_email_context`: dict containing values that are unique for this email but the same
for all recipients of this email. This dict is to be used to fill in slots in email
template. It does not include 'name' and 'email', which will be provided by the to_list.
* `subtask_status` : object of class SubtaskStatus representing current status.
Sends to all addresses contained in to_list that are not also in the Optout table.
Emails are sent multi-part, in both plain text and html.
Returns a tuple of two values:
* First value is a SubtaskStatus object which represents current progress at the end of this call.
* Second value is an exception returned by the innards of the method, indicating a fatal error.
In this case, the number of recipients that were not sent have already been added to the
'failed' count above.
"""
# Get information from current task's request:
parent_task_id = InstructorTask.objects.get(pk=entry_id).task_id
task_id = subtask_status.task_id
total_recipients = len(to_list)
recipient_num = 0
total_recipients_successful = 0
total_recipients_failed = 0
recipients_info = Counter()
log.info(
u"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, TotalRecipients: %s",
parent_task_id, task_id, email_id, total_recipients)
try:
course_email = CourseEmail.objects.get(id=email_id)
except CourseEmail.DoesNotExist as exc:
log.exception(
u"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Could not find email to send.",
parent_task_id, task_id, email_id)
raise
# Exclude optouts (if not a retry):
# Note that we don't have to do the optout logic at all if this is a retry,
# because we have presumably already performed the optout logic on the first
# attempt. Anyone on the to_list on a retry has already passed the filter
# that existed at that time, and we don't need to keep checking for changes
# in the Optout list.
if subtask_status.get_retry_count() == 0:
to_list, num_optout = _filter_optouts_from_recipients(
to_list, course_email.course_id)
subtask_status.increment(skipped=num_optout)
course_title = global_email_context['course_title']
course_language = global_email_context['course_language']
# use the email from address in the CourseEmail, if it is present, otherwise compute it
from_addr = course_email.from_addr if course_email.from_addr else \
_get_source_address(course_email.course_id, course_title, course_language)
# use the CourseEmailTemplate that was associated with the CourseEmail
course_email_template = course_email.get_template()
try:
connection = get_connection()
connection.open()
# Define context values to use in all course emails:
email_context = {'name': '', 'email': ''}
email_context.update(global_email_context)
start_time = time.time()
while to_list:
# Update context with user-specific values from the user at the end of the list.
# At the end of processing this user, they will be popped off of the to_list.
# That way, the to_list will always contain the recipients remaining to be emailed.
# This is convenient for retries, which will need to send to those who haven't
# yet been emailed, but not send to those who have already been sent to.
recipient_num += 1
current_recipient = to_list[-1]
email = current_recipient['email']
if _has_non_ascii_characters(email):
to_list.pop()
total_recipients_failed += 1
log.info(
u"BulkEmail ==> Email address %s contains non-ascii characters. Skipping sending "
u"email to %s, EmailId: %s ", email,
current_recipient['profile__name'], email_id)
subtask_status.increment(failed=1)
continue
email_context['email'] = email
email_context['name'] = current_recipient['profile__name']
email_context['user_id'] = current_recipient['pk']
email_context['course_id'] = course_email.course_id
# Construct message content using templates and context:
plaintext_msg = course_email_template.render_plaintext(
course_email.text_message, email_context)
html_msg = course_email_template.render_htmltext(
course_email.html_message, email_context)
# Create email:
email_msg = EmailMultiAlternatives(course_email.subject,
plaintext_msg,
from_addr, [email],
connection=connection)
email_msg.attach_alternative(html_msg, 'text/html')
# Throttle if we have gotten the rate limiter. This is not very high-tech,
# but if a task has been retried for rate-limiting reasons, then we sleep
# for a period of time between all emails within this task. Choice of
# the value depends on the number of workers that might be sending email in
# parallel, and what the SES throttle rate is.
if subtask_status.retried_nomax > 0:
sleep(settings.BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS)
try:
log.info(
u"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Recipient num: %s/%s, \
Recipient name: %s, Email address: %s", parent_task_id,
task_id, email_id, recipient_num, total_recipients,
current_recipient['profile__name'], email)
connection.send_messages([email_msg])
except SMTPDataError as exc:
# According to SMTP spec, we'll retry error codes in the 4xx range. 5xx range indicates hard failure.
total_recipients_failed += 1
log.error(
u"BulkEmail ==> Status: Failed(SMTPDataError), Task: %s, SubTask: %s, EmailId: %s, \
Recipient num: %s/%s, Email address: %s", parent_task_id,
task_id, email_id, recipient_num, total_recipients, email)
if exc.smtp_code >= 400 and exc.smtp_code < 500:
# This will cause the outer handler to catch the exception and retry the entire task.
raise exc
else:
# This will fall through and not retry the message.
log.warning(
u'BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Recipient num: %s/%s, \
Email not delivered to %s due to error %s',
parent_task_id, task_id, email_id, recipient_num,
total_recipients, email, exc.smtp_error)
subtask_status.increment(failed=1)
except SINGLE_EMAIL_FAILURE_ERRORS as exc:
# This will fall through and not retry the message.
total_recipients_failed += 1
log.error(
u"BulkEmail ==> Status: Failed(SINGLE_EMAIL_FAILURE_ERRORS), Task: %s, SubTask: %s, \
EmailId: %s, Recipient num: %s/%s, Email address: %s, Exception: %s",
parent_task_id, task_id, email_id, recipient_num,
total_recipients, email, exc)
subtask_status.increment(failed=1)
else:
total_recipients_successful += 1
log.info(
u"BulkEmail ==> Status: Success, Task: %s, SubTask: %s, EmailId: %s, \
Recipient num: %s/%s, Email address: %s,", parent_task_id,
task_id, email_id, recipient_num, total_recipients, email)
if settings.BULK_EMAIL_LOG_SENT_EMAILS:
log.info(u'Email with id %s sent to %s', email_id, email)
else:
log.debug(u'Email with id %s sent to %s', email_id, email)
subtask_status.increment(succeeded=1)
# Pop the user that was emailed off the end of the list only once they have
# successfully been processed. (That way, if there were a failure that
# needed to be retried, the user is still on the list.)
recipients_info[email] += 1
to_list.pop()
log.info(
u"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Total Successful Recipients: %s/%s, \
Failed Recipients: %s/%s, Time Taken: %s", parent_task_id, task_id,
email_id, total_recipients_successful, total_recipients,
total_recipients_failed, total_recipients,
time.time() - start_time)
duplicate_recipients = [
u"{0} ({1})".format(email, repetition)
for email, repetition in recipients_info.most_common()
if repetition > 1
]
if duplicate_recipients:
log.info(
u"BulkEmail ==> Task: %s, SubTask: %s, EmailId: %s, Total Duplicate Recipients [%s]: [%s]",
parent_task_id, task_id, email_id, len(duplicate_recipients),
', '.join(duplicate_recipients))
except INFINITE_RETRY_ERRORS as exc:
# Increment the "retried_nomax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_nomax=1, state=RETRY)
return _submit_for_retry(entry_id,
email_id,
to_list,
global_email_context,
exc,
subtask_status,
skip_retry_max=True)
except LIMITED_RETRY_ERRORS as exc:
# Errors caught here cause the email to be retried. The entire task is actually retried
# without popping the current recipient off of the existing list.
# Errors caught are those that indicate a temporary condition that might succeed on retry.
# Increment the "retried_withmax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_withmax=1, state=RETRY)
return _submit_for_retry(entry_id,
email_id,
to_list,
global_email_context,
exc,
subtask_status,
skip_retry_max=False)
except BULK_EMAIL_FAILURE_ERRORS as exc:
num_pending = len(to_list)
log.exception((
u'Task %s: email with id %d caused send_course_email task to fail '
u'with u"fatal" exception. %d emails unsent.'), task_id, email_id,
num_pending)
# Update counters with progress to date, counting unsent emails as failures,
# and set the state to FAILURE:
subtask_status.increment(failed=num_pending, state=FAILURE)
return subtask_status, exc
except Exception as exc: # pylint: disable=broad-except
# Errors caught here cause the email to be retried. The entire task is actually retried
# without popping the current recipient off of the existing list.
# These are unexpected errors. Since they might be due to a temporary condition that might
# succeed on retry, we give them a retry.
log.exception((
u'Task %s: email with id %d caused send_course_email task to fail '
'with unexpected exception. Generating retry.'), task_id,
email_id)
# Increment the "retried_withmax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_withmax=1, state=RETRY)
return _submit_for_retry(entry_id,
email_id,
to_list,
global_email_context,
exc,
subtask_status,
skip_retry_max=False)
else:
# All went well. Update counters with progress to date,
# and set the state to SUCCESS:
subtask_status.increment(state=SUCCESS)
# Successful completion is marked by an exception value of None.
return subtask_status, None
finally:
# Clean up at the end.
connection.close()
