def _get_jwt_builder(self, user, is_client_restricted):
""" Creates and returns a JWTBuilder object for creating JWTs. """
# If JWT scope enforcement is enabled, we need to sign tokens
# given to restricted applications with a key that
# other IDAs do not have access to. This prevents restricted
# applications from getting access to API endpoints available
# on other IDAs which have not yet been protected with the
# scope-related DRF permission classes. Once all endpoints have
# been protected, we can enable all IDAs to use the same new
# (asymmetric) key.
# TODO: ARCH-162
use_asymmetric_key = ENFORCE_JWT_SCOPES.is_enabled(
) and is_client_restricted
monitoring_utils.set_custom_metric('oauth_asymmetric_jwt',
use_asymmetric_key)
log.info("Using Asymmetric JWT: %s", use_asymmetric_key)
return JwtBuilder(
user,
asymmetric=use_asymmetric_key,
secret=settings.JWT_AUTH['JWT_SECRET_KEY'],
issuer=settings.JWT_AUTH['JWT_ISSUER'],
)
def get_schedules_with_target_date_by_bin_and_orgs(
self, order_by='enrollment__user__id'
):
"""
Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list.
Arguments:
order_by -- string for field to sort the resulting Schedules by
"""
target_day = _get_datetime_beginning_of_day(self.target_datetime)
schedule_day_equals_target_day_filter = {
'courseenrollment__schedule__{}__gte'.format(self.schedule_date_field): target_day,
'courseenrollment__schedule__{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1),
}
users = User.objects.filter(
courseenrollment__is_active=True,
**schedule_day_equals_target_day_filter
).annotate(
id_mod=F('id') % self.num_bins
).filter(
id_mod=self.bin_num
)
schedule_day_equals_target_day_filter = {
'{}__gte'.format(self.schedule_date_field): target_day,
'{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1),
}
schedules = Schedule.objects.select_related(
'enrollment__user__profile',
'enrollment__course',
).filter(
Q(enrollment__course__end__isnull=True) | Q(
enrollment__course__end__gte=self.current_datetime
),
self.experience_filter,
enrollment__user__in=users,
enrollment__is_active=True,
active=True,
**schedule_day_equals_target_day_filter
).order_by(order_by)
schedules = self.filter_by_org(schedules)
if "read_replica" in settings.DATABASES:
schedules = schedules.using("read_replica")
LOG.info('Query = %r', schedules.query.sql_with_params())
with function_trace('schedule_query_set_evaluation'):
# This will run the query and cache all of the results in memory.
num_schedules = len(schedules)
LOG.info('Number of schedules = %d', num_schedules)
# This should give us a sense of the volume of data being processed by each task.
set_custom_metric('num_schedules', num_schedules)
return schedules
def get_schedules_with_target_date_by_bin_and_orgs(
self, order_by='enrollment__user__id'
):
"""
Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list.
Arguments:
order_by -- string for field to sort the resulting Schedules by
"""
target_day = _get_datetime_beginning_of_day(self.target_datetime)
schedule_day_equals_target_day_filter = {
'courseenrollment__schedule__{}__gte'.format(self.schedule_date_field): target_day,
'courseenrollment__schedule__{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1),
}
users = User.objects.filter(
courseenrollment__is_active=True,
**schedule_day_equals_target_day_filter
).annotate(
id_mod=F('id') % self.num_bins
).filter(
id_mod=self.bin_num
)
schedule_day_equals_target_day_filter = {
'{}__gte'.format(self.schedule_date_field): target_day,
'{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1),
}
schedules = Schedule.objects.select_related(
'enrollment__user__profile',
'enrollment__course',
).filter(
Q(enrollment__course__end__isnull=True) | Q(
enrollment__course__end__gte=self.current_datetime
),
self.experience_filter,
enrollment__user__in=users,
enrollment__is_active=True,
active=True,
**schedule_day_equals_target_day_filter
).order_by(order_by)
schedules = self.filter_by_org(schedules)
if "read_replica" in settings.DATABASES:
schedules = schedules.using("read_replica")
LOG.info('Query = %r', schedules.query.sql_with_params())
with function_trace('schedule_query_set_evaluation'):
# This will run the query and cache all of the results in memory.
num_schedules = len(schedules)
LOG.info('Number of schedules = %d', num_schedules)
# This should give us a sense of the volume of data being processed by each task.
set_custom_metric('num_schedules', num_schedules)
return schedules
def _recurring_nudge_schedule_send(site_id, msg_str):
site = Site.objects.get(pk=site_id)
if not ScheduleConfig.current(site).deliver_recurring_nudge:
LOG.debug('Recurring Nudge: Message delivery disabled for site %s', site.domain)
return
msg = Message.from_string(msg_str)
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', msg.send_uuid)
# A unique identifier for this particular message.
set_custom_metric('uuid', msg.uuid)
LOG.debug('Recurring Nudge: Sending message = %s', msg_str)
ace.send(msg)
def _recurring_nudge_schedule_send(site_id, msg_str):
site = Site.objects.get(pk=site_id)
if not ScheduleConfig.current(site).deliver_recurring_nudge:
LOG.debug('Recurring Nudge: Message delivery disabled for site %s',
site.domain)
return
msg = Message.from_string(msg_str)
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', msg.send_uuid)
# A unique identifier for this particular message.
set_custom_metric('uuid', msg.uuid)
LOG.debug('Recurring Nudge: Sending message = %s', msg_str)
ace.send(msg)
def dispatch(self, request, *args, **kwargs):
response = super(AccessTokenView,
self).dispatch(request, *args, **kwargs)
token_type = request.POST.get('token_type',
'no_token_type_supplied').lower()
monitoring_utils.set_custom_metric('oauth_token_type', token_type)
monitoring_utils.set_custom_metric('oauth_grant_type',
request.POST.get('grant_type', ''))
if response.status_code == 200 and token_type == 'jwt':
response.content = self._build_jwt_response_from_access_token_response(
request, response)
return response
def get_adapter(self, request):
"""
Returns the appropriate adapter based on the OAuth client linked to the request.
"""
client_id = self._get_client_id(request)
monitoring_utils.set_custom_metric('oauth_client_id', client_id)
if dot_models.Application.objects.filter(client_id=client_id).exists():
monitoring_utils.set_custom_metric('oauth_adapter', 'dot')
return self.dot_adapter
else:
monitoring_utils.set_custom_metric('oauth_adapter', 'dop')
return self.dop_adapter
def _recalculate_subsection_grade(self, **kwargs):
"""
Updates a saved subsection grade.
Keyword Arguments:
user_id (int): id of applicable User object
anonymous_user_id (int, OPTIONAL): Anonymous ID of the User
course_id (string): identifying the course
usage_id (string): identifying the course block
only_if_higher (boolean): indicating whether grades should
be updated only if the new raw_earned is higher than the
previous value.
expected_modified_time (serialized timestamp): indicates when the task
was queued so that we can verify the underlying data update.
score_deleted (boolean): indicating whether the grade change is
a result of the problem's score being deleted.
event_transaction_id (string): uuid identifying the current
event transaction.
event_transaction_type (string): human-readable type of the
event at the root of the current event transaction.
score_db_table (ScoreDatabaseTableEnum): database table that houses
the changed score. Used in conjunction with expected_modified_time.
"""
try:
course_key = CourseLocator.from_string(kwargs['course_id'])
scored_block_usage_key = UsageKey.from_string(
kwargs['usage_id']).replace(course_key=course_key)
set_custom_metrics_for_course_key(course_key)
set_custom_metric('usage_id', unicode(scored_block_usage_key))
# The request cache is not maintained on celery workers,
# where this code runs. So we take the values from the
# main request cache and store them in the local request
# cache. This correlates model-level grading events with
# higher-level ones.
set_event_transaction_id(kwargs.get('event_transaction_id'))
set_event_transaction_type(kwargs.get('event_transaction_type'))
# Verify the database has been updated with the scores when the task was
# created. This race condition occurs if the transaction in the task
# creator's process hasn't committed before the task initiates in the worker
# process.
has_database_updated = _has_db_updated_with_new_score(
self, scored_block_usage_key, **kwargs)
if not has_database_updated:
raise DatabaseNotReadyError
_update_subsection_grades(
course_key,
scored_block_usage_key,
kwargs['only_if_higher'],
kwargs['user_id'],
kwargs['score_deleted'],
)
except Exception as exc: # pylint: disable=broad-except
if not isinstance(exc, KNOWN_RETRY_ERRORS):
log.info(
"tnl-6244 grades unexpected failure: {}. task id: {}. kwargs={}"
.format(
repr(exc),
self.request.id,
kwargs,
))
raise self.retry(kwargs=kwargs, exc=exc)
def _annonate_send_task_for_monitoring(msg):
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', msg.send_uuid)
# A unique identifier for this particular message.
set_custom_metric('uuid', msg.uuid)
def _annotate_for_monitoring(message_type, site, bin_num, target_day_str,
day_offset):
# This identifies the type of message being sent, for example: schedules.recurring_nudge3.
set_custom_metric(
'message_name', '{0}.{1}'.format(message_type.app_label,
message_type.name))
# The domain name of the site we are sending the message for.
set_custom_metric('site', site.domain)
# This is the "bin" of data being processed. We divide up the work into chunks so that we don't tie up celery
# workers for too long. This could help us identify particular bins that are problematic.
set_custom_metric('bin', bin_num)
# The date we are processing data for.
set_custom_metric('target_day', target_day_str)
# The number of days relative to the current date to process data for.
set_custom_metric('day_offset', day_offset)
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', message_type.uuid)
def _annonate_send_task_for_monitoring(msg):
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', msg.send_uuid)
# A unique identifier for this particular message.
set_custom_metric('uuid', msg.uuid)
def _annotate_for_monitoring(message_type, site, bin_num, target_day_str, day_offset):
# This identifies the type of message being sent, for example: schedules.recurring_nudge3.
set_custom_metric('message_name', '{0}.{1}'.format(message_type.app_label, message_type.name))
# The domain name of the site we are sending the message for.
set_custom_metric('site', site.domain)
# This is the "bin" of data being processed. We divide up the work into chunks so that we don't tie up celery
# workers for too long. This could help us identify particular bins that are problematic.
set_custom_metric('bin', bin_num)
# The date we are processing data for.
set_custom_metric('target_day', target_day_str)
# The number of days relative to the current date to process data for.
set_custom_metric('day_offset', day_offset)
# A unique identifier for this batch of messages being sent.
set_custom_metric('send_uuid', message_type.uuid)
def _recalculate_subsection_grade(self, **kwargs):
"""
Updates a saved subsection grade.
Keyword Arguments:
user_id (int): id of applicable User object
anonymous_user_id (int, OPTIONAL): Anonymous ID of the User
course_id (string): identifying the course
usage_id (string): identifying the course block
only_if_higher (boolean): indicating whether grades should
be updated only if the new raw_earned is higher than the
previous value.
expected_modified_time (serialized timestamp): indicates when the task
was queued so that we can verify the underlying data update.
score_deleted (boolean): indicating whether the grade change is
a result of the problem's score being deleted.
event_transaction_id (string): uuid identifying the current
event transaction.
event_transaction_type (string): human-readable type of the
event at the root of the current event transaction.
score_db_table (ScoreDatabaseTableEnum): database table that houses
the changed score. Used in conjunction with expected_modified_time.
"""
try:
course_key = CourseLocator.from_string(kwargs['course_id'])
scored_block_usage_key = UsageKey.from_string(kwargs['usage_id']).replace(course_key=course_key)
set_custom_metrics_for_course_key(course_key)
set_custom_metric('usage_id', unicode(scored_block_usage_key))
# The request cache is not maintained on celery workers,
# where this code runs. So we take the values from the
# main request cache and store them in the local request
# cache. This correlates model-level grading events with
# higher-level ones.
set_event_transaction_id(kwargs.get('event_transaction_id'))
set_event_transaction_type(kwargs.get('event_transaction_type'))
# Verify the database has been updated with the scores when the task was
# created. This race condition occurs if the transaction in the task
# creator's process hasn't committed before the task initiates in the worker
# process.
has_database_updated = _has_db_updated_with_new_score(self, scored_block_usage_key, **kwargs)
if not has_database_updated:
raise DatabaseNotReadyError
_update_subsection_grades(
course_key,
scored_block_usage_key,
kwargs['only_if_higher'],
kwargs['user_id'],
kwargs['score_deleted'],
)
except Exception as exc: # pylint: disable=broad-except
if not isinstance(exc, KNOWN_RETRY_ERRORS):
log.info("tnl-6244 grades unexpected failure: {}. task id: {}. kwargs={}".format(
repr(exc),
self.request.id,
kwargs,
))
raise self.retry(kwargs=kwargs, exc=exc)
def get_schedules_with_target_date_by_bin_and_orgs(schedule_date_field, current_datetime, target_datetime, bin_num,
num_bins=DEFAULT_NUM_BINS, org_list=None, exclude_orgs=False,
order_by='enrollment__user__id'):
"""
Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list.
Arguments:
schedule_date_field -- string field name to query on the User's Schedule model
current_datetime -- datetime that will be used as "right now" in the query
target_datetime -- datetime that the User's Schedule's schedule_date_field value should fall under
bin_num -- int for selecting the bin of Users whose id % num_bins == bin_num
num_bin -- int specifying the number of bins to separate the Users into (default: DEFAULT_NUM_BINS)
org_list -- list of course_org names (strings) that the returned Schedules must or must not be in (default: None)
exclude_orgs -- boolean indicating whether the returned Schedules should exclude (True) the course_orgs in org_list
or strictly include (False) them (default: False)
order_by -- string for field to sort the resulting Schedules by
"""
target_day = _get_datetime_beginning_of_day(target_datetime)
schedule_day_equals_target_day_filter = {
'courseenrollment__schedule__{}__gte'.format(schedule_date_field): target_day,
'courseenrollment__schedule__{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1),
}
users = User.objects.filter(
courseenrollment__is_active=True,
**schedule_day_equals_target_day_filter
).annotate(
id_mod=F('id') % num_bins
).filter(
id_mod=bin_num
)
schedule_day_equals_target_day_filter = {
'{}__gte'.format(schedule_date_field): target_day,
'{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1),
}
schedules = Schedule.objects.select_related(
'enrollment__user__profile',
'enrollment__course',
).prefetch_related(
'enrollment__course__modes'
).filter(
Q(enrollment__course__end__isnull=True) | Q(enrollment__course__end__gte=current_datetime),
enrollment__user__in=users,
enrollment__is_active=True,
**schedule_day_equals_target_day_filter
).order_by(order_by)
if org_list is not None:
if exclude_orgs:
schedules = schedules.exclude(enrollment__course__org__in=org_list)
else:
schedules = schedules.filter(enrollment__course__org__in=org_list)
if "read_replica" in settings.DATABASES:
schedules = schedules.using("read_replica")
LOG.debug('Query = %r', schedules.query.sql_with_params())
with function_trace('schedule_query_set_evaluation'):
# This will run the query and cache all of the results in memory.
num_schedules = len(schedules)
# This should give us a sense of the volume of data being processed by each task.
set_custom_metric('num_schedules', num_schedules)
return schedules
def get_schedules_with_target_date_by_bin_and_orgs(
schedule_date_field,
current_datetime,
target_datetime,
bin_num,
num_bins=DEFAULT_NUM_BINS,
org_list=None,
exclude_orgs=False,
order_by='enrollment__user__id'):
"""
Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list.
Arguments:
schedule_date_field -- string field name to query on the User's Schedule model
current_datetime -- datetime that will be used as "right now" in the query
target_datetime -- datetime that the User's Schedule's schedule_date_field value should fall under
bin_num -- int for selecting the bin of Users whose id % num_bins == bin_num
num_bin -- int specifying the number of bins to separate the Users into (default: DEFAULT_NUM_BINS)
org_list -- list of course_org names (strings) that the returned Schedules must or must not be in (default: None)
exclude_orgs -- boolean indicating whether the returned Schedules should exclude (True) the course_orgs in org_list
or strictly include (False) them (default: False)
order_by -- string for field to sort the resulting Schedules by
"""
target_day = _get_datetime_beginning_of_day(target_datetime)
schedule_day_equals_target_day_filter = {
'courseenrollment__schedule__{}__gte'.format(schedule_date_field):
target_day,
'courseenrollment__schedule__{}__lt'.format(schedule_date_field):
target_day + datetime.timedelta(days=1),
}
users = User.objects.filter(
courseenrollment__is_active=True,
**schedule_day_equals_target_day_filter).annotate(
id_mod=F('id') % num_bins).filter(id_mod=bin_num)
schedule_day_equals_target_day_filter = {
'{}__gte'.format(schedule_date_field):
target_day,
'{}__lt'.format(schedule_date_field):
target_day + datetime.timedelta(days=1),
}
schedules = Schedule.objects.select_related(
'enrollment__user__profile',
'enrollment__course',
).prefetch_related('enrollment__course__modes').filter(
Q(enrollment__course__end__isnull=True)
| Q(enrollment__course__end__gte=current_datetime),
enrollment__user__in=users,
enrollment__is_active=True,
**schedule_day_equals_target_day_filter).order_by(order_by)
if org_list is not None:
if exclude_orgs:
schedules = schedules.exclude(enrollment__course__org__in=org_list)
else:
schedules = schedules.filter(enrollment__course__org__in=org_list)
if "read_replica" in settings.DATABASES:
schedules = schedules.using("read_replica")
LOG.debug('Query = %r', schedules.query.sql_with_params())
with function_trace('schedule_query_set_evaluation'):
# This will run the query and cache all of the results in memory.
num_schedules = len(schedules)
# This should give us a sense of the volume of data being processed by each task.
set_custom_metric('num_schedules', num_schedules)
return schedules
