def get_context_data(self, **kwargs): context = super(CourseView, self).get_context_data(**kwargs) self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=settings.LMS_DEFAULT_TIMEOUT) self.course = self.client.courses(self.course_id) return context
def _update_active_students(course_key, section_data): auth_token = settings.ANALYTICS_DATA_TOKEN base_url = settings.ANALYTICS_DATA_URL section_data['active_student_count'] = 'N/A' section_data['active_student_count_start'] = 'N/A' section_data['active_student_count_end'] = 'N/A' try: client = Client(base_url=base_url, auth_token=auth_token) course = client.courses(unicode(course_key)) recent_activity = course.recent_activity() section_data['active_student_count'] = recent_activity['count'] def format_date(value): return value.split('T')[0] start = recent_activity['interval_start'] end = recent_activity['interval_end'] section_data['active_student_count_start'] = format_date(start) section_data['active_student_count_end'] = format_date(end) except (ClientError, KeyError) as e: log.exception(e)
def __init__(self, course_id, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id)
def test_strip_trailing_slash(self): url = 'http://example.com' client = Client(url) self.assertEqual(client.base_url, url) url_with_slash = 'http://example.com/' client = Client(url_with_slash) self.assertEqual(client.base_url, url)
def test_failed_authentication(self): client = Client(base_url=self.api_url, auth_token='atoken') httpretty.register_uri(httpretty.GET, self.test_url, body='', status=401) self.assertEqual(client.has_resource(self.test_endpoint), False) self.assertEqual(httpretty.last_request().headers['Authorization'], 'Token atoken')
def get_sequential_open_distrib(course_id, enrollment): """ Returns the number of students that opened each subsection/sequential of the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Outputs a dict mapping the 'module_id' to the number of students that have opened that subsection/sequential. """ sequential_open_distrib = {} non_student_list = get_non_student_list(course_id) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for "opening a subsection" data queryset = models.StudentModule.objects.filter( course_id__exact=course_id, module_type__exact='sequential', ).exclude(student_id__in=non_student_list).values( 'module_state_key').annotate( count_sequential=Count('module_state_key')) for row in queryset: module_id = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) sequential_open_distrib[module_id] = row['count_sequential'] else: # Retrieve course object down to subsection course = modulestore().get_course(course_id, depth=2) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): module = client.modules(course_id, subsection.location) try: sequential_open = module.sequential_open_distribution() except NotFoundError: pass else: sequential_open_distrib[ subsection.location] = sequential_open[0]['count'] return sequential_open_distrib
class CourseView(LoginRequiredMixin, CoursePermissionMixin, TemplateView): """ Base course view. Adds conveniences such as course_id attribute, and handles 404s when retrieving data from the API. """ client = None course = None course_id = None user = None def dispatch(self, request, *args, **kwargs): self.user = request.user self.course_id = kwargs['course_id'] try: return super(CourseView, self).dispatch(request, *args, **kwargs) except NotFoundError: raise Http404 def get_context_data(self, **kwargs): context = super(CourseView, self).get_context_data(**kwargs) self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=5) self.course = self.client.courses(self.course_id) return context
def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) api_version = request.GET.get('v', '0') analytics_base_url = settings.DATA_API_URL_V1 if api_version == '1' else settings.DATA_API_URL self.analytics_client = Client(base_url=analytics_base_url, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT)
def api_client(self): analytics_api_client = Client(base_url=self.partner.analytics_url, auth_token=self.partner.analytics_token, timeout=self.API_TIMEOUT) return analytics_api_client
class BasePresenter(object): """ This is the base class for the pages and sets up the analytics client for the presenters to use to access the data API. """ def __init__(self, course_id, timeout=5): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id) @staticmethod def parse_api_date(s): """ Parse a string according to the API date format. """ return datetime.datetime.strptime(s, Client.DATE_FORMAT).date() @staticmethod def parse_api_datetime(s): """ Parse a string according to the API datetime format. """ return datetime.datetime.strptime(s, Client.DATETIME_FORMAT) @staticmethod def strip_time(s): return s[:-7]
class BasePresenter(object): """ This is the base class for the pages and sets up the analytics client for the presenters to use to access the data API. """ def __init__(self, course_id, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id) def get_current_date(self): return datetime.datetime.utcnow().strftime(Client.DATE_FORMAT) @staticmethod def parse_api_date(s): """ Parse a string according to the API date format. """ return datetime.datetime.strptime(s, Client.DATE_FORMAT).date() @staticmethod def parse_api_datetime(s): """ Parse a string according to the API datetime format. """ return datetime.datetime.strptime(s, Client.DATETIME_FORMAT) @staticmethod def strip_time(s): return s[:-7] @staticmethod def sum_counts(data): return sum(datum['count'] for datum in data)
class CourseView(LoginRequiredMixin, CourseValidMixin, CoursePermissionMixin, TemplateView): """ Base course view. Adds conveniences such as course_id attribute, and handles 404s when retrieving data from the API. """ client = None course = None course_id = None course_key = None user = None def dispatch(self, request, *args, **kwargs): self.user = request.user self.course_id = request.course_id self.course_key = request.course_key # some views will catch the NotFoundError to set data to a state that # the template can rendering a loading error message for the section try: return super(CourseView, self).dispatch(request, *args, **kwargs) except NotFoundError as e: logger.error('The requested data from the Analytics Data API was not found: %s', e) raise Http404 except ClientError as e: logger.error('An error occurred while retrieving data from the Analytics Data API: %s', e) raise def get_context_data(self, **kwargs): context = super(CourseView, self).get_context_data(**kwargs) self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=5) self.course = self.client.courses(self.course_id) return context
def main(): start = time.time() api_client = Client(base_url=API_SERVER_URL, auth_token=API_AUTH_TOKEN, timeout=1000) http_client = requests.Session() if BASIC_AUTH_CREDENTIALS: http_client.auth = BASIC_AUTH_CREDENTIALS login(http_client) # Basic auth is no longer needed http_client.auth = None # Get courses courses = get_courses(http_client) # Collect the data reports = Queue() try: p = Pool(NUM_PROCESSES, pool_init, [reports, api_client, http_client.cookies]) p.map(check_course, courses) except Exception as e: # pylint: disable=broad-except logger.error('Validation failed to finish: %s', e) # Write the data to an external file write_csv(reports) end = time.time() logger.info('Finished in %d seconds.', end - start)
def health(_request): OK = 'OK' UNAVAILABLE = 'UNAVAILABLE' overall_status = analytics_api_status = database_status = UNAVAILABLE try: cursor = connection.cursor() cursor.execute("SELECT 1") cursor.fetchone() cursor.close() database_status = OK except DatabaseError: # pylint: disable=catching-non-exception database_status = UNAVAILABLE try: client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN) if client.status.healthy: analytics_api_status = OK except ClientError as e: logger.exception('API is not reachable from dashboard: %s', e) analytics_api_status = UNAVAILABLE overall_status = OK if (analytics_api_status == database_status == OK) else UNAVAILABLE data = { 'overall_status': overall_status, 'detailed_status': { 'database_connection': database_status, 'analytics_api': analytics_api_status } } return HttpResponse(json.dumps(data), content_type='application/json', status=200 if overall_status == OK else 503)
class BasePresenter_snail2(object): def __init__(self, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL_SNAIL2, auth_token=settings.DATA_API_AUTH_TOKEN_SNAIL, timeout=timeout) def get_current_date(self): return datetime.datetime.utcnow().strftime(Client.DATE_FORMAT) @staticmethod def parse_api_date(s): """ Parse a string according to the API date format. """ return datetime.datetime.strptime(s, Client.DATE_FORMAT).date() @staticmethod def parse_api_datetime(s): """ Parse a string according to the API datetime format. """ return datetime.datetime.strptime(s, Client.DATETIME_FORMAT) @staticmethod def strip_time(s): return s[:-7] @staticmethod def sum_counts(data): return sum(datum['count'] for datum in data) def risk_response(self): return self.client.get('sparkml/atriskstudents/')
def setUp(self): super().setUp() api_url = API_SERVER_URL auth_token = API_AUTH_TOKEN self.analytics_api_client = Client(api_url, auth_token=auth_token, timeout=10)
def test_get_programs(self, program_ids, course_ids): ''''Test programs filtered from API response.''' presenter = ProgramsPresenter(Client('base_url')) with mock.patch('analyticsclient.programs.Programs.programs', mock.Mock(return_value=self.mock_api_response)): actual_programs = presenter.get_programs(program_ids=program_ids, course_ids=course_ids) self.assertListEqual(actual_programs, self.get_expected_programs(program_ids=program_ids, course_ids=course_ids))
def __init__(self, course_id, http_cookies=None): self.course_id = course_id self.analytics_api_client = Client(base_url=API_SERVER_URL, auth_token=API_AUTH_TOKEN, timeout=1000) self.course_api_client = CourseStructureApiClient( COURSE_API_URL, COURSE_API_KEY, 5) self.http_client = requests.Session() self.http_client.cookies = http_cookies
def test_no_summaries(self): cache.clear() # previous test has course_ids=None case cached presenter = CourseSummariesPresenter(Client('base_url')) with mock.patch( 'analyticsclient.course_summaries.CourseSummaries.course_summaries', mock.Mock(return_value=[])): summaries, last_updated = presenter.get_course_summaries() self.assertListEqual(summaries, []) self.assertIsNone(last_updated)
def get_sequential_open_distrib(course_id, enrollment): """ Returns the number of students that opened each subsection/sequential of the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Outputs a dict mapping the 'module_id' to the number of students that have opened that subsection/sequential. """ sequential_open_distrib = {} non_student_list = get_non_student_list(course_id) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for "opening a subsection" data queryset = models.StudentModule.objects.filter( course_id__exact=course_id, module_type__exact='sequential', ).exclude(student_id__in=non_student_list).values('module_state_key').annotate(count_sequential=Count('module_state_key')) for row in queryset: module_id = course_id.make_usage_key_from_deprecated_string(row['module_state_key']) sequential_open_distrib[module_id] = row['count_sequential'] else: # Retrieve course object down to subsection course = modulestore().get_course(course_id, depth=2) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): module = client.modules(course_id, subsection.location) try: sequential_open = module.sequential_open_distribution() except NotFoundError: pass else: sequential_open_distrib[subsection.location] = sequential_open[0]['count'] return sequential_open_distrib
def test_get_course_summary_metrics(self): presenter = CourseSummariesPresenter(Client('base_url')) metrics = presenter.get_course_summary_metrics( self._PRESENTER_SUMMARIES.values()) expected = { 'total_enrollment': 5111, 'current_enrollment': 3888, 'enrollment_change_7_days': 4, 'verified_enrollment': 13, 'masters_enrollment': 1111, } self.assertEqual(metrics, expected)
def health(_request): if newrelic: # pragma: no cover newrelic.agent.ignore_transaction() overall_status = analytics_api_status = database_status = UNAVAILABLE try: cursor = connection.cursor() cursor.execute("SELECT 1") cursor.fetchone() cursor.close() database_status = OK except DatabaseError as e: logger.exception('Insights database is not reachable: %s', e) database_status = UNAVAILABLE try: client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=0.35) # Note: client.status.healthy sends a request to the health endpoint on # the Analytics API. The request may throw a TimeoutError. Currently, # other exceptions are caught by the client.status.healthy method # itself, which will return False in those cases. analytics_api_healthy = client.status.healthy except TimeoutError as e: logger.exception( 'Analytics API health check timed out from dashboard: %s', e) analytics_api_status = UNAVAILABLE else: if analytics_api_healthy: analytics_api_status = OK else: logger.error('Analytics API health check failed from dashboard') analytics_api_status = UNAVAILABLE overall_status = OK if ( analytics_api_status == database_status == OK) else UNAVAILABLE data = { 'overall_status': overall_status, 'detailed_status': { 'database_connection': database_status, 'analytics_api': analytics_api_status } } return HttpResponse(json.dumps(data), content_type='application/json', status=200 if overall_status == OK else 503)
def test_get_summaries(self, input_course_ids, ouptut_course_ids): presenter = CourseSummariesPresenter(Client('base_url')) if input_course_ids: mock_api_response = [ self._API_SUMMARIES[course_id] for course_id in input_course_ids ] else: mock_api_response = list(self._API_SUMMARIES.values()) expected_summaries = [ self._PRESENTER_SUMMARIES[course_id] for course_id in ouptut_course_ids ] with mock.patch( 'analyticsclient.course_summaries.CourseSummaries.course_summaries', mock.Mock(return_value=mock_api_response)): actual_summaries, last_updated = presenter.get_course_summaries( course_ids=input_course_ids) for actual, expected in zip(actual_summaries, expected_summaries): self.assertCountEqual(actual, expected) self.assertEqual(last_updated, utils.CREATED_DATETIME)
def __init__(self, course_id, timeout=5): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id)
class LearnerAnalyticsView(View): """ Displays the Learner Analytics Dashboard. """ def __init__(self): View.__init__(self) self.analytics_client = Client(base_url=settings.ANALYTICS_API_URL, auth_token=settings.ANALYTICS_API_KEY) @method_decorator(login_required) @method_decorator(cache_control(no_cache=True, no_store=True, must_revalidate=True)) @method_decorator(ensure_valid_course_key) def get(self, request, course_id): """ Displays the user's Learner Analytics for the specified course. Arguments: request: HTTP request course_id (unicode): course id """ course_key = CourseKey.from_string(course_id) if not ENABLE_DASHBOARD_TAB.is_enabled(course_key): raise Http404 course = get_course_with_access(request.user, 'load', course_key, check_if_enrolled=True) course_url_name = default_course_url_name(course.id) course_url = reverse(course_url_name, kwargs={'course_id': unicode(course.id)}) is_verified = CourseEnrollment.is_enrolled_as_verified(request.user, course_key) has_access = is_verified or request.user.is_staff enrollment = CourseEnrollment.get_enrollment(request.user, course_key) upgrade_price = None upgrade_url = None if enrollment and enrollment.upgrade_deadline: upgrade_url = EcommerceService().upgrade_url(request.user, course_key) upgrade_price = get_cosmetic_verified_display_price(course) context = { 'upgrade_price': upgrade_price, 'upgrade_link': upgrade_url, 'course': course, 'course_url': course_url, 'disable_courseware_js': True, 'uses_pattern_library': True, 'is_self_paced': course.self_paced, 'is_verified': is_verified, 'has_access': has_access, } if (has_access): grading_policy = course.grading_policy (raw_grade_data, answered_percent, percent_grade) = self.get_grade_data(request.user, course_key, grading_policy['GRADE_CUTOFFS']) raw_schedule_data = self.get_assignments_with_due_date(request, course_key) grade_data, schedule_data = self.sort_grade_and_schedule_data(raw_grade_data, raw_schedule_data) # TODO: LEARNER-3854: Fix hacked defaults with real error handling if implementing Learner Analytics. try: weekly_active_users = self.get_weekly_course_activity_count(course_key) week_streak = self.consecutive_weeks_of_course_activity_for_user( request.user.username, course_key ) except Exception as e: logging.exception(e) weekly_active_users = 134 week_streak = 1 context.update({ 'grading_policy': grading_policy, 'assignment_grades': grade_data, 'answered_percent': answered_percent, 'assignment_schedule': schedule_data, 'assignment_schedule_raw': raw_schedule_data, 'profile_image_urls': get_profile_image_urls_for_user(request.user, request), 'discussion_info': self.get_discussion_data(request, course_key), 'passing_grade': math.ceil(100 * course.lowest_passing_grade), 'percent_grade': math.ceil(100 * percent_grade), 'weekly_active_users': weekly_active_users, 'week_streak': week_streak, }) return render_to_response('learner_analytics/dashboard.html', context) def get_grade_data(self, user, course_key, grade_cutoffs): """ Collects and formats the grades data for a particular user and course. Args: user (User) course_key (CourseKey) grade_cutoffs: # TODO: LEARNER-3854: Complete docstring if implementing Learner Analytics. """ course_grade = CourseGradeFactory().read(user, course_key=course_key) grades = [] total_earned = 0 total_possible = 0 # answered_percent seems to be unused and it does not take into account assignment type weightings answered_percent = None chapter_grades = course_grade.chapter_grades.values() for chapter in chapter_grades: # Note: this code exists on the progress page. We should be able to remove it going forward. if not chapter['display_name'] == "hidden": for subsection_grade in chapter['sections']: log.info(subsection_grade.display_name) possible = subsection_grade.graded_total.possible earned = subsection_grade.graded_total.earned passing_grade = math.ceil(possible * grade_cutoffs['Pass']) grades.append({ 'assignment_type': subsection_grade.format, 'total_earned': earned, 'total_possible': possible, 'passing_grade': passing_grade, 'display_name': subsection_grade.display_name, 'location': unicode(subsection_grade.location), 'assigment_url': reverse('jump_to_id', kwargs={ 'course_id': unicode(course_key), 'module_id': unicode(subsection_grade.location), }) }) if earned > 0: total_earned += earned total_possible += possible if total_possible > 0: answered_percent = float(total_earned) / total_possible return (grades, answered_percent, course_grade.percent) def sort_grade_and_schedule_data(self, grade_data, schedule_data): """ Sort the assignments in grade_data and schedule_data to be in the same order. """ schedule_dict = {assignment['location']: assignment for assignment in schedule_data} sorted_schedule_data = [] sorted_grade_data = [] for grade in grade_data: assignment = schedule_dict.get(grade['location']) if assignment: sorted_grade_data.append(grade) sorted_schedule_data.append(assignment) return sorted_grade_data, sorted_schedule_data def get_discussion_data(self, request, course_key): """ Collects and formats the discussion data from a particular user and course. Args: request (HttpRequest) course_key (CourseKey) """ try: context = create_user_profile_context(request, course_key, request.user.id) except Exception as e: # TODO: LEARNER-3854: Clean-up error handling if continuing support. return { 'content_authored': 0, 'thread_votes': 0, } threads = context['threads'] profiled_user = context['profiled_user'] # TODO: LEARNER-3854: If implementing Learner Analytics, rename to content_authored_count. content_authored = profiled_user['threads_count'] + profiled_user['comments_count'] thread_votes = 0 for thread in threads: if thread['user_id'] == profiled_user['external_id']: thread_votes += thread['votes']['count'] discussion_data = { 'content_authored': content_authored, 'thread_votes': thread_votes, } return discussion_data def get_assignments_with_due_date(self, request, course_key): """ Returns a list of assignment (graded) blocks with due dates, including due date and location. Args: request (HttpRequest) course_key (CourseKey) """ course_usage_key = modulestore().make_course_usage_key(course_key) all_blocks = get_blocks( request, course_usage_key, user=request.user, nav_depth=3, requested_fields=['display_name', 'due', 'graded', 'format'], block_types_filter=['sequential'] ) assignment_blocks = [] for (location, block) in all_blocks['blocks'].iteritems(): if block.get('graded', False): assignment_blocks.append(block) block['due'] = block['due'].isoformat() if block.get('due') is not None else None block['location'] = unicode(location) return assignment_blocks def get_weekly_course_activity_count(self, course_key): """ Get the count of any course activity (total for all users) from previous 7 days. Args: course_key (CourseKey) """ cache_key = 'learner_analytics_{course_key}_weekly_activities'.format(course_key=course_key) activities = cache.get(cache_key) if not activities: log.info('Weekly course activities for course {course_key} was not cached - fetching from Analytics API' .format(course_key=course_key)) weekly_course_activities = self.analytics_client.courses(course_key).activity() if not weekly_course_activities or 'any' not in weekly_course_activities[0]: return 0 # weekly course activities should only have one item activities = weekly_course_activities[0] cache.set(cache_key, activities, LearnerAnalyticsView.seconds_to_cache_expiration()) return activities['any'] def consecutive_weeks_of_course_activity_for_user(self, username, course_key): """ Get the most recent count of consecutive days that a user has performed a course activity Args: username (str) course_key (CourseKey) """ cache_key = 'learner_analytics_{username}_{course_key}_engagement_timeline'\ .format(username=username, course_key=course_key) timeline = cache.get(cache_key) if not timeline: log.info('Engagement timeline for course {course_key} was not cached - fetching from Analytics API' .format(course_key=course_key)) # TODO (LEARNER-3470): @jaebradley replace this once the Analytics client has an engagement timeline method url = '{base_url}/engagement_timelines/{username}?course_id={course_key}'\ .format(base_url=settings.ANALYTICS_API_URL, username=username, course_key=urllib.quote_plus(unicode(course_key))) headers = {'Authorization': 'Token {token}'.format(token=settings.ANALYTICS_API_KEY)} response = requests.get(url=url, headers=headers) data = response.json() if not data or 'days' not in data or not data['days']: return 0 # Analytics API returns data in ascending (by date) order - we want to count starting from most recent day data_ordered_by_date_descending = list(reversed(data['days'])) cache.set(cache_key, data_ordered_by_date_descending, LearnerAnalyticsView.seconds_to_cache_expiration()) timeline = data_ordered_by_date_descending return LearnerAnalyticsView.calculate_week_streak(timeline) @staticmethod def calculate_week_streak(daily_activities): """ Check number of weeks in a row that a user has performed some activity. Regardless of when a week starts, a sufficient condition for checking if a specific week had any user activity (given a list of daily activities ordered by date) is to iterate through the list of days 7 days at a time and check to see if any of those days had any activity. Args: daily_activities: sorted list of dictionaries containing activities and their counts """ week_streak = 0 seven_day_buckets = [daily_activities[i:i + 7] for i in range(0, len(daily_activities), 7)] for bucket in seven_day_buckets: if any(LearnerAnalyticsView.has_activity(day) for day in bucket): week_streak += 1 else: return week_streak return week_streak @staticmethod def has_activity(daily_activity): """ Validate that a course had some activity that day Args: daily_activity: dictionary of activities and their counts """ return int(daily_activity['problems_attempted']) > 0 \ or int(daily_activity['problems_completed']) > 0 \ or int(daily_activity['discussion_contributions']) > 0 \ or int(daily_activity['videos_viewed']) > 0 @staticmethod def seconds_to_cache_expiration(): """Calculate cache expiration seconds. Currently set to seconds until midnight UTC""" next_midnight_utc = (datetime.today() + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.utc) now_utc = datetime.now(tz=pytz.utc) return round((next_midnight_utc - now_utc).total_seconds())
def __init__(self): View.__init__(self) self.analytics_client = Client(base_url=settings.ANALYTICS_API_URL, auth_token=settings.ANALYTICS_API_KEY)
class LearnerAnalyticsView(View): """ Displays the Learner Analytics Dashboard. """ def __init__(self): View.__init__(self) self.analytics_client = Client(base_url=settings.ANALYTICS_API_URL, auth_token=settings.ANALYTICS_API_KEY) @method_decorator(login_required) @method_decorator( cache_control(no_cache=True, no_store=True, must_revalidate=True)) @method_decorator(ensure_valid_course_key) def get(self, request, course_id): """ Displays the user's Learner Analytics for the specified course. Arguments: request: HTTP request course_id (unicode): course id """ course_key = CourseKey.from_string(course_id) if not ENABLE_DASHBOARD_TAB.is_enabled(course_key): raise Http404 course = get_course_with_access(request.user, 'load', course_key, check_if_enrolled=True) course_url_name = default_course_url_name(course.id) course_url = reverse(course_url_name, kwargs={'course_id': unicode(course.id)}) is_verified = CourseEnrollment.is_enrolled_as_verified( request.user, course_key) has_access = is_verified or request.user.is_staff enrollment = CourseEnrollment.get_enrollment(request.user, course_key) upgrade_price = None upgrade_url = None if enrollment and enrollment.upgrade_deadline: upgrade_url = EcommerceService().upgrade_url( request.user, course_key) upgrade_price = get_cosmetic_verified_display_price(course) context = { 'upgrade_price': upgrade_price, 'upgrade_link': upgrade_url, 'course': course, 'course_url': course_url, 'disable_courseware_js': True, 'uses_pattern_library': True, 'is_self_paced': course.self_paced, 'is_verified': is_verified, 'has_access': has_access, } if (has_access): grading_policy = course.grading_policy (raw_grade_data, answered_percent, percent_grade) = self.get_grade_data( request.user, course_key, grading_policy['GRADE_CUTOFFS']) raw_schedule_data = self.get_assignments_with_due_date( request, course_key) grade_data, schedule_data = self.sort_grade_and_schedule_data( raw_grade_data, raw_schedule_data) # TODO: LEARNER-3854: Fix hacked defaults with real error handling if implementing Learner Analytics. try: weekly_active_users = self.get_weekly_course_activity_count( course_key) week_streak = self.consecutive_weeks_of_course_activity_for_user( request.user.username, course_key) except Exception as e: logging.exception(e) weekly_active_users = 134 week_streak = 1 context.update({ 'grading_policy': grading_policy, 'assignment_grades': grade_data, 'answered_percent': answered_percent, 'assignment_schedule': schedule_data, 'assignment_schedule_raw': raw_schedule_data, 'profile_image_urls': get_profile_image_urls_for_user(request.user, request), 'discussion_info': self.get_discussion_data(request, course_key), 'passing_grade': math.ceil(100 * course.lowest_passing_grade), 'percent_grade': math.ceil(100 * percent_grade), 'weekly_active_users': weekly_active_users, 'week_streak': week_streak, }) return render_to_response('learner_analytics/dashboard.html', context) def get_grade_data(self, user, course_key, grade_cutoffs): """ Collects and formats the grades data for a particular user and course. Args: user (User) course_key (CourseKey) grade_cutoffs: # TODO: LEARNER-3854: Complete docstring if implementing Learner Analytics. """ course_grade = CourseGradeFactory().read(user, course_key=course_key) grades = [] total_earned = 0 total_possible = 0 # answered_percent seems to be unused and it does not take into account assignment type weightings answered_percent = None chapter_grades = course_grade.chapter_grades.values() for chapter in chapter_grades: # Note: this code exists on the progress page. We should be able to remove it going forward. if not chapter['display_name'] == "hidden": for subsection_grade in chapter['sections']: log.info(subsection_grade.display_name) possible = subsection_grade.graded_total.possible earned = subsection_grade.graded_total.earned passing_grade = math.ceil(possible * grade_cutoffs['Pass']) grades.append({ 'assignment_type': subsection_grade.format, 'total_earned': earned, 'total_possible': possible, 'passing_grade': passing_grade, 'display_name': subsection_grade.display_name, 'location': unicode(subsection_grade.location), 'assigment_url': reverse('jump_to_id', kwargs={ 'course_id': unicode(course_key), 'module_id': unicode(subsection_grade.location), }) }) if earned > 0: total_earned += earned total_possible += possible if total_possible > 0: answered_percent = float(total_earned) / total_possible return (grades, answered_percent, course_grade.percent) def sort_grade_and_schedule_data(self, grade_data, schedule_data): """ Sort the assignments in grade_data and schedule_data to be in the same order. """ schedule_dict = { assignment['location']: assignment for assignment in schedule_data } sorted_schedule_data = [] sorted_grade_data = [] for grade in grade_data: assignment = schedule_dict.get(grade['location']) if assignment: sorted_grade_data.append(grade) sorted_schedule_data.append(assignment) return sorted_grade_data, sorted_schedule_data def get_discussion_data(self, request, course_key): """ Collects and formats the discussion data from a particular user and course. Args: request (HttpRequest) course_key (CourseKey) """ try: context = create_user_profile_context(request, course_key, request.user.id) except Exception as e: # TODO: LEARNER-3854: Clean-up error handling if continuing support. return { 'content_authored': 0, 'thread_votes': 0, } threads = context['threads'] profiled_user = context['profiled_user'] # TODO: LEARNER-3854: If implementing Learner Analytics, rename to content_authored_count. content_authored = profiled_user['threads_count'] + profiled_user[ 'comments_count'] thread_votes = 0 for thread in threads: if thread['user_id'] == profiled_user['external_id']: thread_votes += thread['votes']['count'] discussion_data = { 'content_authored': content_authored, 'thread_votes': thread_votes, } return discussion_data def get_assignments_with_due_date(self, request, course_key): """ Returns a list of assignment (graded) blocks with due dates, including due date and location. Args: request (HttpRequest) course_key (CourseKey) """ course_usage_key = modulestore().make_course_usage_key(course_key) all_blocks = get_blocks( request, course_usage_key, user=request.user, nav_depth=3, requested_fields=['display_name', 'due', 'graded', 'format'], block_types_filter=['sequential']) assignment_blocks = [] for (location, block) in all_blocks['blocks'].iteritems(): if block.get('graded', False): assignment_blocks.append(block) block['due'] = block['due'].isoformat() if block.get( 'due') is not None else None block['location'] = unicode(location) return assignment_blocks def get_weekly_course_activity_count(self, course_key): """ Get the count of any course activity (total for all users) from previous 7 days. Args: course_key (CourseKey) """ cache_key = 'learner_analytics_{course_key}_weekly_activities'.format( course_key=course_key) activities = cache.get(cache_key) if not activities: log.info( 'Weekly course activities for course {course_key} was not cached - fetching from Analytics API' .format(course_key=course_key)) weekly_course_activities = self.analytics_client.courses( course_key).activity() if not weekly_course_activities or 'any' not in weekly_course_activities[ 0]: return 0 # weekly course activities should only have one item activities = weekly_course_activities[0] cache.set(cache_key, activities, LearnerAnalyticsView.seconds_to_cache_expiration()) return activities['any'] def consecutive_weeks_of_course_activity_for_user(self, username, course_key): """ Get the most recent count of consecutive days that a user has performed a course activity Args: username (str) course_key (CourseKey) """ cache_key = 'learner_analytics_{username}_{course_key}_engagement_timeline'\ .format(username=username, course_key=course_key) timeline = cache.get(cache_key) if not timeline: log.info( 'Engagement timeline for course {course_key} was not cached - fetching from Analytics API' .format(course_key=course_key)) # TODO (LEARNER-3470): @jaebradley replace this once the Analytics client has an engagement timeline method url = '{base_url}/engagement_timelines/{username}?course_id={course_key}'\ .format(base_url=settings.ANALYTICS_API_URL, username=username, course_key=urllib.quote_plus(unicode(course_key))) headers = { 'Authorization': 'Token {token}'.format(token=settings.ANALYTICS_API_KEY) } response = requests.get(url=url, headers=headers) data = response.json() if not data or 'days' not in data or not data['days']: return 0 # Analytics API returns data in ascending (by date) order - we want to count starting from most recent day data_ordered_by_date_descending = list(reversed(data['days'])) cache.set(cache_key, data_ordered_by_date_descending, LearnerAnalyticsView.seconds_to_cache_expiration()) timeline = data_ordered_by_date_descending return LearnerAnalyticsView.calculate_week_streak(timeline) @staticmethod def calculate_week_streak(daily_activities): """ Check number of weeks in a row that a user has performed some activity. Regardless of when a week starts, a sufficient condition for checking if a specific week had any user activity (given a list of daily activities ordered by date) is to iterate through the list of days 7 days at a time and check to see if any of those days had any activity. Args: daily_activities: sorted list of dictionaries containing activities and their counts """ week_streak = 0 seven_day_buckets = [ daily_activities[i:i + 7] for i in range(0, len(daily_activities), 7) ] for bucket in seven_day_buckets: if any(LearnerAnalyticsView.has_activity(day) for day in bucket): week_streak += 1 else: return week_streak return week_streak @staticmethod def has_activity(daily_activity): """ Validate that a course had some activity that day Args: daily_activity: dictionary of activities and their counts """ return int(daily_activity['problems_attempted']) > 0 \ or int(daily_activity['problems_completed']) > 0 \ or int(daily_activity['discussion_contributions']) > 0 \ or int(daily_activity['videos_viewed']) > 0 @staticmethod def seconds_to_cache_expiration(): """Calculate cache expiration seconds. Currently set to seconds until midnight UTC""" next_midnight_utc = (datetime.today() + timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.utc) now_utc = datetime.now(tz=pytz.utc) return round((next_midnight_utc - now_utc).total_seconds())
def get_problem_set_grade_distrib(course_id, problem_set, enrollment): """ Returns the grade distribution for the problems specified in `problem_set`. `course_id` the course ID for the course interested in `problem_set` an array of UsageKeys representing problem module_id's. `enrollment` the number of students enrolled in this course. Requests from the database the a count of each grade for each problem in the `problem_set`. Returns a dict, where the key is the problem 'module_id' and the value is a dict with two parts: 'max_grade' - the maximum grade possible for the course 'grade_distrib' - array of tuples (`grade`,`count`) ordered by `grade` """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for set of problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, module_state_key__in=problem_set, ).exclude(student_id__in=non_student_list).values( 'module_state_key', 'grade', 'max_grade', ).annotate(count_grade=Count('grade')).order_by('module_state_key', 'grade') # Loop through resultset building data for each problem for row in queryset: problem_id = course_id.make_usage_key_from_deprecated_string(row['module_state_key']) if problem_id not in prob_grade_distrib: prob_grade_distrib[problem_id] = { 'max_grade': 0, 'grade_distrib': [], } curr_grade_distrib = prob_grade_distrib[problem_id] curr_grade_distrib['grade_distrib'].append((row['grade'], row['count_grade'])) if curr_grade_distrib['max_grade'] < row['max_grade']: curr_grade_distrib['max_grade'] = row['max_grade'] else: # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for problem in problem_set: module = client.modules(course_id, problem) try: grade_distribution = module.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: if problem in prob_grade_distrib: if prob_grade_distrib[problem]['max_grade'] < score['max_grade']: prob_grade_distrib[problem]['max_grade'] = score['max_grade'] prob_grade_distrib[problem]['grade_distrib'].append((score['grade'], score['count'])) else: prob_grade_distrib[problem] = { 'max_grade': score['max_grade'], 'grade_distrib': [(score['grade'], score['count'])], } return prob_grade_distrib
def get_problem_grade_distribution(course_id, enrollment): """ Returns the grade distribution per problem for the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Output is 2 dicts: 'prob-grade_distrib' where the key is the problem 'module_id' and the value is a dict with: 'max_grade' - max grade for this problem 'grade_distrib' - array of tuples (`grade`,`count`). 'total_student_count' where the key is problem 'module_id' and the value is number of students attempting the problem """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} total_student_count = defaultdict(int) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for all problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, ).exclude(student_id__in=non_student_list).values('module_state_key', 'grade', 'max_grade').annotate(count_grade=Count('grade')) # Loop through resultset building data for each problem for row in queryset: curr_problem = course_id.make_usage_key_from_deprecated_string(row['module_state_key']) # Build set of grade distributions for each problem that has student responses if curr_problem in prob_grade_distrib: prob_grade_distrib[curr_problem]['grade_distrib'].append((row['grade'], row['count_grade'])) if ((prob_grade_distrib[curr_problem]['max_grade'] != row['max_grade']) and (prob_grade_distrib[curr_problem]['max_grade'] < row['max_grade'])): prob_grade_distrib[curr_problem]['max_grade'] = row['max_grade'] else: prob_grade_distrib[curr_problem] = { 'max_grade': row['max_grade'], 'grade_distrib': [(row['grade'], row['count_grade']), ], } # Build set of total students attempting each problem total_student_count[curr_problem] += row['count_grade'] else: # Retrieve course object down to problems course = modulestore().get_course(course_id, depth=4) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): for unit in subsection.get_children(): for child in unit.get_children(): if child.location.category not in PROB_TYPE_LIST: continue problem_id = child.location problem = client.modules(course_id, problem_id) try: grade_distribution = problem.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: total_student_count[problem_id] += score['count'] if problem_id in prob_grade_distrib: if prob_grade_distrib[problem_id]['max_grade'] < score['max_grade']: prob_grade_distrib[problem_id]['max_grade'] = score['max_grade'] prob_grade_distrib[problem_id]['grade_distrib'].append((score['grade'], score['count'])) else: prob_grade_distrib[problem_id] = { 'max_grade': score['max_grade'], 'grade_distrib': [(score['grade'], score['count']), ], } return prob_grade_distrib, total_student_count
def test_date_format(self): self.assertEqual(Client.DATE_FORMAT, '%Y-%m-%d') self.assertEqual(Client('').DATE_FORMAT, '%Y-%m-%d')
def setUp(self): """Configure Client.""" self.api_url = 'http://localhost:9999/api/v1' self.client = Client(self.api_url)
def get_context_data(self, **kwargs): context = super(CourseView, self).get_context_data(**kwargs) self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=5) self.course = self.client.courses(self.course_id) return context
def get_problem_set_grade_distrib(course_id, problem_set, enrollment): """ Returns the grade distribution for the problems specified in `problem_set`. `course_id` the course ID for the course interested in `problem_set` an array of UsageKeys representing problem module_id's. `enrollment` the number of students enrolled in this course. Requests from the database the a count of each grade for each problem in the `problem_set`. Returns a dict, where the key is the problem 'module_id' and the value is a dict with two parts: 'max_grade' - the maximum grade possible for the course 'grade_distrib' - array of tuples (`grade`,`count`) ordered by `grade` """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for set of problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, module_state_key__in=problem_set, ).exclude(student_id__in=non_student_list).values( 'module_state_key', 'grade', 'max_grade', ).annotate(count_grade=Count('grade')).order_by( 'module_state_key', 'grade') # Loop through resultset building data for each problem for row in queryset: problem_id = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) if problem_id not in prob_grade_distrib: prob_grade_distrib[problem_id] = { 'max_grade': 0, 'grade_distrib': [], } curr_grade_distrib = prob_grade_distrib[problem_id] curr_grade_distrib['grade_distrib'].append( (row['grade'], row['count_grade'])) if curr_grade_distrib['max_grade'] < row['max_grade']: curr_grade_distrib['max_grade'] = row['max_grade'] else: # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for problem in problem_set: module = client.modules(course_id, problem) try: grade_distribution = module.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: if problem in prob_grade_distrib: if prob_grade_distrib[problem]['max_grade'] < score[ 'max_grade']: prob_grade_distrib[problem]['max_grade'] = score[ 'max_grade'] prob_grade_distrib[problem]['grade_distrib'].append( (score['grade'], score['count'])) else: prob_grade_distrib[problem] = { 'max_grade': score['max_grade'], 'grade_distrib': [(score['grade'], score['count'])], } return prob_grade_distrib
def get_problem_grade_distribution(course_id, enrollment): """ Returns the grade distribution per problem for the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Output is 2 dicts: 'prob-grade_distrib' where the key is the problem 'module_id' and the value is a dict with: 'max_grade' - max grade for this problem 'grade_distrib' - array of tuples (`grade`,`count`). 'total_student_count' where the key is problem 'module_id' and the value is number of students attempting the problem """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} total_student_count = defaultdict(int) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for all problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, ).exclude(student_id__in=non_student_list).values( 'module_state_key', 'grade', 'max_grade').annotate(count_grade=Count('grade')) # Loop through resultset building data for each problem for row in queryset: curr_problem = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) # Build set of grade distributions for each problem that has student responses if curr_problem in prob_grade_distrib: prob_grade_distrib[curr_problem]['grade_distrib'].append( (row['grade'], row['count_grade'])) if ((prob_grade_distrib[curr_problem]['max_grade'] != row['max_grade']) and (prob_grade_distrib[curr_problem]['max_grade'] < row['max_grade'])): prob_grade_distrib[curr_problem]['max_grade'] = row[ 'max_grade'] else: prob_grade_distrib[curr_problem] = { 'max_grade': row['max_grade'], 'grade_distrib': [ (row['grade'], row['count_grade']), ], } # Build set of total students attempting each problem total_student_count[curr_problem] += row['count_grade'] else: # Retrieve course object down to problems course = modulestore().get_course(course_id, depth=4) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): for unit in subsection.get_children(): for child in unit.get_children(): if child.location.category not in PROB_TYPE_LIST: continue problem_id = child.location problem = client.modules(course_id, problem_id) try: grade_distribution = problem.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: total_student_count[problem_id] += score['count'] if problem_id in prob_grade_distrib: if prob_grade_distrib[problem_id][ 'max_grade'] < score['max_grade']: prob_grade_distrib[problem_id][ 'max_grade'] = score['max_grade'] prob_grade_distrib[problem_id][ 'grade_distrib'].append( (score['grade'], score['count'])) else: prob_grade_distrib[problem_id] = { 'max_grade': score['max_grade'], 'grade_distrib': [ (score['grade'], score['count']), ], } return prob_grade_distrib, total_student_count
def __init__(self, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL_SNAIL2, auth_token=settings.DATA_API_AUTH_TOKEN_SNAIL, timeout=timeout)