def _get_jwt_builder(self, user, is_client_restricted): """ Creates and returns a JWTBuilder object for creating JWTs. """ # If JWT scope enforcement is enabled, we need to sign tokens # given to restricted applications with a key that # other IDAs do not have access to. This prevents restricted # applications from getting access to API endpoints available # on other IDAs which have not yet been protected with the # scope-related DRF permission classes. Once all endpoints have # been protected, we can enable all IDAs to use the same new # (asymmetric) key. # TODO: ARCH-162 use_asymmetric_key = ENFORCE_JWT_SCOPES.is_enabled( ) and is_client_restricted monitoring_utils.set_custom_metric('oauth_asymmetric_jwt', use_asymmetric_key) log.info("Using Asymmetric JWT: %s", use_asymmetric_key) return JwtBuilder( user, asymmetric=use_asymmetric_key, secret=settings.JWT_AUTH['JWT_SECRET_KEY'], issuer=settings.JWT_AUTH['JWT_ISSUER'], )
def get_schedules_with_target_date_by_bin_and_orgs( self, order_by='enrollment__user__id' ): """ Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list. Arguments: order_by -- string for field to sort the resulting Schedules by """ target_day = _get_datetime_beginning_of_day(self.target_datetime) schedule_day_equals_target_day_filter = { 'courseenrollment__schedule__{}__gte'.format(self.schedule_date_field): target_day, 'courseenrollment__schedule__{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1), } users = User.objects.filter( courseenrollment__is_active=True, **schedule_day_equals_target_day_filter ).annotate( id_mod=F('id') % self.num_bins ).filter( id_mod=self.bin_num ) schedule_day_equals_target_day_filter = { '{}__gte'.format(self.schedule_date_field): target_day, '{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1), } schedules = Schedule.objects.select_related( 'enrollment__user__profile', 'enrollment__course', ).filter( Q(enrollment__course__end__isnull=True) | Q( enrollment__course__end__gte=self.current_datetime ), self.experience_filter, enrollment__user__in=users, enrollment__is_active=True, active=True, **schedule_day_equals_target_day_filter ).order_by(order_by) schedules = self.filter_by_org(schedules) if "read_replica" in settings.DATABASES: schedules = schedules.using("read_replica") LOG.info('Query = %r', schedules.query.sql_with_params()) with function_trace('schedule_query_set_evaluation'): # This will run the query and cache all of the results in memory. num_schedules = len(schedules) LOG.info('Number of schedules = %d', num_schedules) # This should give us a sense of the volume of data being processed by each task. set_custom_metric('num_schedules', num_schedules) return schedules
def get_schedules_with_target_date_by_bin_and_orgs( self, order_by='enrollment__user__id' ): """ Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list. Arguments: order_by -- string for field to sort the resulting Schedules by """ target_day = _get_datetime_beginning_of_day(self.target_datetime) schedule_day_equals_target_day_filter = { 'courseenrollment__schedule__{}__gte'.format(self.schedule_date_field): target_day, 'courseenrollment__schedule__{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1), } users = User.objects.filter( courseenrollment__is_active=True, **schedule_day_equals_target_day_filter ).annotate( id_mod=F('id') % self.num_bins ).filter( id_mod=self.bin_num ) schedule_day_equals_target_day_filter = { '{}__gte'.format(self.schedule_date_field): target_day, '{}__lt'.format(self.schedule_date_field): target_day + datetime.timedelta(days=1), } schedules = Schedule.objects.select_related( 'enrollment__user__profile', 'enrollment__course', ).filter( Q(enrollment__course__end__isnull=True) | Q( enrollment__course__end__gte=self.current_datetime ), self.experience_filter, enrollment__user__in=users, enrollment__is_active=True, active=True, **schedule_day_equals_target_day_filter ).order_by(order_by) schedules = self.filter_by_org(schedules) if "read_replica" in settings.DATABASES: schedules = schedules.using("read_replica") LOG.info('Query = %r', schedules.query.sql_with_params()) with function_trace('schedule_query_set_evaluation'): # This will run the query and cache all of the results in memory. num_schedules = len(schedules) LOG.info('Number of schedules = %d', num_schedules) # This should give us a sense of the volume of data being processed by each task. set_custom_metric('num_schedules', num_schedules) return schedules
def _recurring_nudge_schedule_send(site_id, msg_str): site = Site.objects.get(pk=site_id) if not ScheduleConfig.current(site).deliver_recurring_nudge: LOG.debug('Recurring Nudge: Message delivery disabled for site %s', site.domain) return msg = Message.from_string(msg_str) # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', msg.send_uuid) # A unique identifier for this particular message. set_custom_metric('uuid', msg.uuid) LOG.debug('Recurring Nudge: Sending message = %s', msg_str) ace.send(msg)
def _recurring_nudge_schedule_send(site_id, msg_str): site = Site.objects.get(pk=site_id) if not ScheduleConfig.current(site).deliver_recurring_nudge: LOG.debug('Recurring Nudge: Message delivery disabled for site %s', site.domain) return msg = Message.from_string(msg_str) # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', msg.send_uuid) # A unique identifier for this particular message. set_custom_metric('uuid', msg.uuid) LOG.debug('Recurring Nudge: Sending message = %s', msg_str) ace.send(msg)
def dispatch(self, request, *args, **kwargs): response = super(AccessTokenView, self).dispatch(request, *args, **kwargs) token_type = request.POST.get('token_type', 'no_token_type_supplied').lower() monitoring_utils.set_custom_metric('oauth_token_type', token_type) monitoring_utils.set_custom_metric('oauth_grant_type', request.POST.get('grant_type', '')) if response.status_code == 200 and token_type == 'jwt': response.content = self._build_jwt_response_from_access_token_response( request, response) return response
def get_adapter(self, request): """ Returns the appropriate adapter based on the OAuth client linked to the request. """ client_id = self._get_client_id(request) monitoring_utils.set_custom_metric('oauth_client_id', client_id) if dot_models.Application.objects.filter(client_id=client_id).exists(): monitoring_utils.set_custom_metric('oauth_adapter', 'dot') return self.dot_adapter else: monitoring_utils.set_custom_metric('oauth_adapter', 'dop') return self.dop_adapter
def _recalculate_subsection_grade(self, **kwargs): """ Updates a saved subsection grade. Keyword Arguments: user_id (int): id of applicable User object anonymous_user_id (int, OPTIONAL): Anonymous ID of the User course_id (string): identifying the course usage_id (string): identifying the course block only_if_higher (boolean): indicating whether grades should be updated only if the new raw_earned is higher than the previous value. expected_modified_time (serialized timestamp): indicates when the task was queued so that we can verify the underlying data update. score_deleted (boolean): indicating whether the grade change is a result of the problem's score being deleted. event_transaction_id (string): uuid identifying the current event transaction. event_transaction_type (string): human-readable type of the event at the root of the current event transaction. score_db_table (ScoreDatabaseTableEnum): database table that houses the changed score. Used in conjunction with expected_modified_time. """ try: course_key = CourseLocator.from_string(kwargs['course_id']) scored_block_usage_key = UsageKey.from_string( kwargs['usage_id']).replace(course_key=course_key) set_custom_metrics_for_course_key(course_key) set_custom_metric('usage_id', unicode(scored_block_usage_key)) # The request cache is not maintained on celery workers, # where this code runs. So we take the values from the # main request cache and store them in the local request # cache. This correlates model-level grading events with # higher-level ones. set_event_transaction_id(kwargs.get('event_transaction_id')) set_event_transaction_type(kwargs.get('event_transaction_type')) # Verify the database has been updated with the scores when the task was # created. This race condition occurs if the transaction in the task # creator's process hasn't committed before the task initiates in the worker # process. has_database_updated = _has_db_updated_with_new_score( self, scored_block_usage_key, **kwargs) if not has_database_updated: raise DatabaseNotReadyError _update_subsection_grades( course_key, scored_block_usage_key, kwargs['only_if_higher'], kwargs['user_id'], kwargs['score_deleted'], ) except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, KNOWN_RETRY_ERRORS): log.info( "tnl-6244 grades unexpected failure: {}. task id: {}. kwargs={}" .format( repr(exc), self.request.id, kwargs, )) raise self.retry(kwargs=kwargs, exc=exc)
def _annonate_send_task_for_monitoring(msg): # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', msg.send_uuid) # A unique identifier for this particular message. set_custom_metric('uuid', msg.uuid)
def _annotate_for_monitoring(message_type, site, bin_num, target_day_str, day_offset): # This identifies the type of message being sent, for example: schedules.recurring_nudge3. set_custom_metric( 'message_name', '{0}.{1}'.format(message_type.app_label, message_type.name)) # The domain name of the site we are sending the message for. set_custom_metric('site', site.domain) # This is the "bin" of data being processed. We divide up the work into chunks so that we don't tie up celery # workers for too long. This could help us identify particular bins that are problematic. set_custom_metric('bin', bin_num) # The date we are processing data for. set_custom_metric('target_day', target_day_str) # The number of days relative to the current date to process data for. set_custom_metric('day_offset', day_offset) # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', message_type.uuid)
def _annonate_send_task_for_monitoring(msg): # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', msg.send_uuid) # A unique identifier for this particular message. set_custom_metric('uuid', msg.uuid)
def _annotate_for_monitoring(message_type, site, bin_num, target_day_str, day_offset): # This identifies the type of message being sent, for example: schedules.recurring_nudge3. set_custom_metric('message_name', '{0}.{1}'.format(message_type.app_label, message_type.name)) # The domain name of the site we are sending the message for. set_custom_metric('site', site.domain) # This is the "bin" of data being processed. We divide up the work into chunks so that we don't tie up celery # workers for too long. This could help us identify particular bins that are problematic. set_custom_metric('bin', bin_num) # The date we are processing data for. set_custom_metric('target_day', target_day_str) # The number of days relative to the current date to process data for. set_custom_metric('day_offset', day_offset) # A unique identifier for this batch of messages being sent. set_custom_metric('send_uuid', message_type.uuid)
def _recalculate_subsection_grade(self, **kwargs): """ Updates a saved subsection grade. Keyword Arguments: user_id (int): id of applicable User object anonymous_user_id (int, OPTIONAL): Anonymous ID of the User course_id (string): identifying the course usage_id (string): identifying the course block only_if_higher (boolean): indicating whether grades should be updated only if the new raw_earned is higher than the previous value. expected_modified_time (serialized timestamp): indicates when the task was queued so that we can verify the underlying data update. score_deleted (boolean): indicating whether the grade change is a result of the problem's score being deleted. event_transaction_id (string): uuid identifying the current event transaction. event_transaction_type (string): human-readable type of the event at the root of the current event transaction. score_db_table (ScoreDatabaseTableEnum): database table that houses the changed score. Used in conjunction with expected_modified_time. """ try: course_key = CourseLocator.from_string(kwargs['course_id']) scored_block_usage_key = UsageKey.from_string(kwargs['usage_id']).replace(course_key=course_key) set_custom_metrics_for_course_key(course_key) set_custom_metric('usage_id', unicode(scored_block_usage_key)) # The request cache is not maintained on celery workers, # where this code runs. So we take the values from the # main request cache and store them in the local request # cache. This correlates model-level grading events with # higher-level ones. set_event_transaction_id(kwargs.get('event_transaction_id')) set_event_transaction_type(kwargs.get('event_transaction_type')) # Verify the database has been updated with the scores when the task was # created. This race condition occurs if the transaction in the task # creator's process hasn't committed before the task initiates in the worker # process. has_database_updated = _has_db_updated_with_new_score(self, scored_block_usage_key, **kwargs) if not has_database_updated: raise DatabaseNotReadyError _update_subsection_grades( course_key, scored_block_usage_key, kwargs['only_if_higher'], kwargs['user_id'], kwargs['score_deleted'], ) except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, KNOWN_RETRY_ERRORS): log.info("tnl-6244 grades unexpected failure: {}. task id: {}. kwargs={}".format( repr(exc), self.request.id, kwargs, )) raise self.retry(kwargs=kwargs, exc=exc)
def get_schedules_with_target_date_by_bin_and_orgs(schedule_date_field, current_datetime, target_datetime, bin_num, num_bins=DEFAULT_NUM_BINS, org_list=None, exclude_orgs=False, order_by='enrollment__user__id'): """ Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list. Arguments: schedule_date_field -- string field name to query on the User's Schedule model current_datetime -- datetime that will be used as "right now" in the query target_datetime -- datetime that the User's Schedule's schedule_date_field value should fall under bin_num -- int for selecting the bin of Users whose id % num_bins == bin_num num_bin -- int specifying the number of bins to separate the Users into (default: DEFAULT_NUM_BINS) org_list -- list of course_org names (strings) that the returned Schedules must or must not be in (default: None) exclude_orgs -- boolean indicating whether the returned Schedules should exclude (True) the course_orgs in org_list or strictly include (False) them (default: False) order_by -- string for field to sort the resulting Schedules by """ target_day = _get_datetime_beginning_of_day(target_datetime) schedule_day_equals_target_day_filter = { 'courseenrollment__schedule__{}__gte'.format(schedule_date_field): target_day, 'courseenrollment__schedule__{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1), } users = User.objects.filter( courseenrollment__is_active=True, **schedule_day_equals_target_day_filter ).annotate( id_mod=F('id') % num_bins ).filter( id_mod=bin_num ) schedule_day_equals_target_day_filter = { '{}__gte'.format(schedule_date_field): target_day, '{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1), } schedules = Schedule.objects.select_related( 'enrollment__user__profile', 'enrollment__course', ).prefetch_related( 'enrollment__course__modes' ).filter( Q(enrollment__course__end__isnull=True) | Q(enrollment__course__end__gte=current_datetime), enrollment__user__in=users, enrollment__is_active=True, **schedule_day_equals_target_day_filter ).order_by(order_by) if org_list is not None: if exclude_orgs: schedules = schedules.exclude(enrollment__course__org__in=org_list) else: schedules = schedules.filter(enrollment__course__org__in=org_list) if "read_replica" in settings.DATABASES: schedules = schedules.using("read_replica") LOG.debug('Query = %r', schedules.query.sql_with_params()) with function_trace('schedule_query_set_evaluation'): # This will run the query and cache all of the results in memory. num_schedules = len(schedules) # This should give us a sense of the volume of data being processed by each task. set_custom_metric('num_schedules', num_schedules) return schedules
def get_schedules_with_target_date_by_bin_and_orgs( schedule_date_field, current_datetime, target_datetime, bin_num, num_bins=DEFAULT_NUM_BINS, org_list=None, exclude_orgs=False, order_by='enrollment__user__id'): """ Returns Schedules with the target_date, related to Users whose id matches the bin_num, and filtered by org_list. Arguments: schedule_date_field -- string field name to query on the User's Schedule model current_datetime -- datetime that will be used as "right now" in the query target_datetime -- datetime that the User's Schedule's schedule_date_field value should fall under bin_num -- int for selecting the bin of Users whose id % num_bins == bin_num num_bin -- int specifying the number of bins to separate the Users into (default: DEFAULT_NUM_BINS) org_list -- list of course_org names (strings) that the returned Schedules must or must not be in (default: None) exclude_orgs -- boolean indicating whether the returned Schedules should exclude (True) the course_orgs in org_list or strictly include (False) them (default: False) order_by -- string for field to sort the resulting Schedules by """ target_day = _get_datetime_beginning_of_day(target_datetime) schedule_day_equals_target_day_filter = { 'courseenrollment__schedule__{}__gte'.format(schedule_date_field): target_day, 'courseenrollment__schedule__{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1), } users = User.objects.filter( courseenrollment__is_active=True, **schedule_day_equals_target_day_filter).annotate( id_mod=F('id') % num_bins).filter(id_mod=bin_num) schedule_day_equals_target_day_filter = { '{}__gte'.format(schedule_date_field): target_day, '{}__lt'.format(schedule_date_field): target_day + datetime.timedelta(days=1), } schedules = Schedule.objects.select_related( 'enrollment__user__profile', 'enrollment__course', ).prefetch_related('enrollment__course__modes').filter( Q(enrollment__course__end__isnull=True) | Q(enrollment__course__end__gte=current_datetime), enrollment__user__in=users, enrollment__is_active=True, **schedule_day_equals_target_day_filter).order_by(order_by) if org_list is not None: if exclude_orgs: schedules = schedules.exclude(enrollment__course__org__in=org_list) else: schedules = schedules.filter(enrollment__course__org__in=org_list) if "read_replica" in settings.DATABASES: schedules = schedules.using("read_replica") LOG.debug('Query = %r', schedules.query.sql_with_params()) with function_trace('schedule_query_set_evaluation'): # This will run the query and cache all of the results in memory. num_schedules = len(schedules) # This should give us a sense of the volume of data being processed by each task. set_custom_metric('num_schedules', num_schedules) return schedules