def test_strip_trailing_slash(self): url = 'http://example.com' client = Client(url) self.assertEqual(client.base_url, url) url_with_slash = 'http://example.com/' client = Client(url_with_slash) self.assertEqual(client.base_url, url)
def health(_request): OK = 'OK' UNAVAILABLE = 'UNAVAILABLE' overall_status = analytics_api_status = database_status = UNAVAILABLE try: cursor = connection.cursor() cursor.execute("SELECT 1") cursor.fetchone() cursor.close() database_status = OK except DatabaseError: # pylint: disable=catching-non-exception database_status = UNAVAILABLE try: client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN) if client.status.healthy: analytics_api_status = OK except ClientError as e: logger.exception('API is not reachable from dashboard: %s', e) analytics_api_status = UNAVAILABLE overall_status = OK if (analytics_api_status == database_status == OK) else UNAVAILABLE data = { 'overall_status': overall_status, 'detailed_status': { 'database_connection': database_status, 'analytics_api': analytics_api_status } } return HttpResponse(json.dumps(data), content_type='application/json', status=200 if overall_status == OK else 503)
def api_client(self): analytics_api_client = Client(base_url=self.partner.analytics_url, auth_token=self.partner.analytics_token, timeout=self.API_TIMEOUT) return analytics_api_client
def _update_active_students(course_key, section_data): auth_token = settings.ANALYTICS_DATA_TOKEN base_url = settings.ANALYTICS_DATA_URL section_data['active_student_count'] = 'N/A' section_data['active_student_count_start'] = 'N/A' section_data['active_student_count_end'] = 'N/A' try: client = Client(base_url=base_url, auth_token=auth_token) course = client.courses(unicode(course_key)) recent_activity = course.recent_activity() section_data['active_student_count'] = recent_activity['count'] def format_date(value): return value.split('T')[0] start = recent_activity['interval_start'] end = recent_activity['interval_end'] section_data['active_student_count_start'] = format_date(start) section_data['active_student_count_end'] = format_date(end) except (ClientError, KeyError) as e: log.exception(e)
def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) api_version = request.GET.get('v', '0') analytics_base_url = settings.DATA_API_URL_V1 if api_version == '1' else settings.DATA_API_URL self.analytics_client = Client(base_url=analytics_base_url, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT)
def main(): start = time.time() api_client = Client(base_url=API_SERVER_URL, auth_token=API_AUTH_TOKEN, timeout=1000) http_client = requests.Session() if BASIC_AUTH_CREDENTIALS: http_client.auth = BASIC_AUTH_CREDENTIALS login(http_client) # Basic auth is no longer needed http_client.auth = None # Get courses courses = get_courses(http_client) # Collect the data reports = Queue() try: p = Pool(NUM_PROCESSES, pool_init, [reports, api_client, http_client.cookies]) p.map(check_course, courses) except Exception as e: # pylint: disable=broad-except logger.error('Validation failed to finish: %s', e) # Write the data to an external file write_csv(reports) end = time.time() logger.info('Finished in %d seconds.', end - start)
def get_context_data(self, **kwargs): context = super(CourseView, self).get_context_data(**kwargs) self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=settings.LMS_DEFAULT_TIMEOUT) self.course = self.client.courses(self.course_id) return context
def setUp(self): super().setUp() api_url = API_SERVER_URL auth_token = API_AUTH_TOKEN self.analytics_api_client = Client(api_url, auth_token=auth_token, timeout=10)
def __init__(self, course_id, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id)
def test_no_summaries(self): cache.clear() # previous test has course_ids=None case cached presenter = CourseSummariesPresenter(Client('base_url')) with mock.patch( 'analyticsclient.course_summaries.CourseSummaries.course_summaries', mock.Mock(return_value=[])): summaries, last_updated = presenter.get_course_summaries() self.assertListEqual(summaries, []) self.assertIsNone(last_updated)
def test_get_programs(self, program_ids, course_ids): ''''Test programs filtered from API response.''' presenter = ProgramsPresenter(Client('base_url')) with mock.patch('analyticsclient.programs.Programs.programs', mock.Mock(return_value=self.mock_api_response)): actual_programs = presenter.get_programs(program_ids=program_ids, course_ids=course_ids) self.assertListEqual(actual_programs, self.get_expected_programs(program_ids=program_ids, course_ids=course_ids))
def __init__(self, course_id, http_cookies=None): self.course_id = course_id self.analytics_api_client = Client(base_url=API_SERVER_URL, auth_token=API_AUTH_TOKEN, timeout=1000) self.course_api_client = CourseStructureApiClient( COURSE_API_URL, COURSE_API_KEY, 5) self.http_client = requests.Session() self.http_client.cookies = http_cookies
def test_failed_authentication(self): client = Client(base_url=self.api_url, auth_token='atoken') httpretty.register_uri(httpretty.GET, self.test_url, body='', status=401) self.assertEqual(client.has_resource(self.test_endpoint), False) self.assertEqual(httpretty.last_request().headers['Authorization'], 'Token atoken')
def test_get_course_summary_metrics(self): presenter = CourseSummariesPresenter(Client('base_url')) metrics = presenter.get_course_summary_metrics( self._PRESENTER_SUMMARIES.values()) expected = { 'total_enrollment': 5111, 'current_enrollment': 3888, 'enrollment_change_7_days': 4, 'verified_enrollment': 13, 'masters_enrollment': 1111, } self.assertEqual(metrics, expected)
def health(_request): if newrelic: # pragma: no cover newrelic.agent.ignore_transaction() overall_status = analytics_api_status = database_status = UNAVAILABLE try: cursor = connection.cursor() cursor.execute("SELECT 1") cursor.fetchone() cursor.close() database_status = OK except DatabaseError as e: logger.exception('Insights database is not reachable: %s', e) database_status = UNAVAILABLE try: client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=0.35) # Note: client.status.healthy sends a request to the health endpoint on # the Analytics API. The request may throw a TimeoutError. Currently, # other exceptions are caught by the client.status.healthy method # itself, which will return False in those cases. analytics_api_healthy = client.status.healthy except TimeoutError as e: logger.exception( 'Analytics API health check timed out from dashboard: %s', e) analytics_api_status = UNAVAILABLE else: if analytics_api_healthy: analytics_api_status = OK else: logger.error('Analytics API health check failed from dashboard') analytics_api_status = UNAVAILABLE overall_status = OK if ( analytics_api_status == database_status == OK) else UNAVAILABLE data = { 'overall_status': overall_status, 'detailed_status': { 'database_connection': database_status, 'analytics_api': analytics_api_status } } return HttpResponse(json.dumps(data), content_type='application/json', status=200 if overall_status == OK else 503)
def get_sequential_open_distrib(course_id, enrollment): """ Returns the number of students that opened each subsection/sequential of the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Outputs a dict mapping the 'module_id' to the number of students that have opened that subsection/sequential. """ sequential_open_distrib = {} non_student_list = get_non_student_list(course_id) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for "opening a subsection" data queryset = models.StudentModule.objects.filter( course_id__exact=course_id, module_type__exact='sequential', ).exclude(student_id__in=non_student_list).values( 'module_state_key').annotate( count_sequential=Count('module_state_key')) for row in queryset: module_id = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) sequential_open_distrib[module_id] = row['count_sequential'] else: # Retrieve course object down to subsection course = modulestore().get_course(course_id, depth=2) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): module = client.modules(course_id, subsection.location) try: sequential_open = module.sequential_open_distribution() except NotFoundError: pass else: sequential_open_distrib[ subsection.location] = sequential_open[0]['count'] return sequential_open_distrib
def test_get_summaries(self, input_course_ids, ouptut_course_ids): presenter = CourseSummariesPresenter(Client('base_url')) if input_course_ids: mock_api_response = [ self._API_SUMMARIES[course_id] for course_id in input_course_ids ] else: mock_api_response = list(self._API_SUMMARIES.values()) expected_summaries = [ self._PRESENTER_SUMMARIES[course_id] for course_id in ouptut_course_ids ] with mock.patch( 'analyticsclient.course_summaries.CourseSummaries.course_summaries', mock.Mock(return_value=mock_api_response)): actual_summaries, last_updated = presenter.get_course_summaries( course_ids=input_course_ids) for actual, expected in zip(actual_summaries, expected_summaries): self.assertCountEqual(actual, expected) self.assertEqual(last_updated, utils.CREATED_DATETIME)
def __init__(self, course_id, timeout=5): self.client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN, timeout=timeout) self.course_id = course_id self.course = self.client.courses(self.course_id)
def __init__(self): View.__init__(self) self.analytics_client = Client(base_url=settings.ANALYTICS_API_URL, auth_token=settings.ANALYTICS_API_KEY)
def setUp(self): """Configure Client.""" self.api_url = 'http://localhost:9999/api/v1' self.client = Client(self.api_url)
def test_date_format(self): self.assertEqual(Client.DATE_FORMAT, '%Y-%m-%d') self.assertEqual(Client('').DATE_FORMAT, '%Y-%m-%d')
def get_problem_set_grade_distrib(course_id, problem_set, enrollment): """ Returns the grade distribution for the problems specified in `problem_set`. `course_id` the course ID for the course interested in `problem_set` an array of UsageKeys representing problem module_id's. `enrollment` the number of students enrolled in this course. Requests from the database the a count of each grade for each problem in the `problem_set`. Returns a dict, where the key is the problem 'module_id' and the value is a dict with two parts: 'max_grade' - the maximum grade possible for the course 'grade_distrib' - array of tuples (`grade`,`count`) ordered by `grade` """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for set of problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, module_state_key__in=problem_set, ).exclude(student_id__in=non_student_list).values( 'module_state_key', 'grade', 'max_grade', ).annotate(count_grade=Count('grade')).order_by( 'module_state_key', 'grade') # Loop through resultset building data for each problem for row in queryset: problem_id = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) if problem_id not in prob_grade_distrib: prob_grade_distrib[problem_id] = { 'max_grade': 0, 'grade_distrib': [], } curr_grade_distrib = prob_grade_distrib[problem_id] curr_grade_distrib['grade_distrib'].append( (row['grade'], row['count_grade'])) if curr_grade_distrib['max_grade'] < row['max_grade']: curr_grade_distrib['max_grade'] = row['max_grade'] else: # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for problem in problem_set: module = client.modules(course_id, problem) try: grade_distribution = module.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: if problem in prob_grade_distrib: if prob_grade_distrib[problem]['max_grade'] < score[ 'max_grade']: prob_grade_distrib[problem]['max_grade'] = score[ 'max_grade'] prob_grade_distrib[problem]['grade_distrib'].append( (score['grade'], score['count'])) else: prob_grade_distrib[problem] = { 'max_grade': score['max_grade'], 'grade_distrib': [(score['grade'], score['count'])], } return prob_grade_distrib
def __init__(self, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT): self.client = Client(base_url=settings.DATA_API_URL_SNAIL2, auth_token=settings.DATA_API_AUTH_TOKEN_SNAIL, timeout=timeout)
def get_problem_grade_distribution(course_id, enrollment): """ Returns the grade distribution per problem for the course `course_id` the course ID for the course interested in `enrollment` the number of students enrolled in this course. Output is 2 dicts: 'prob-grade_distrib' where the key is the problem 'module_id' and the value is a dict with: 'max_grade' - max grade for this problem 'grade_distrib' - array of tuples (`grade`,`count`). 'total_student_count' where the key is problem 'module_id' and the value is number of students attempting the problem """ non_student_list = get_non_student_list(course_id) prob_grade_distrib = {} total_student_count = defaultdict(int) if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL: # Aggregate query on studentmodule table for grade data for all problems in course queryset = models.StudentModule.objects.filter( course_id__exact=course_id, grade__isnull=False, module_type__in=PROB_TYPE_LIST, ).exclude(student_id__in=non_student_list).values( 'module_state_key', 'grade', 'max_grade').annotate(count_grade=Count('grade')) # Loop through resultset building data for each problem for row in queryset: curr_problem = course_id.make_usage_key_from_deprecated_string( row['module_state_key']) # Build set of grade distributions for each problem that has student responses if curr_problem in prob_grade_distrib: prob_grade_distrib[curr_problem]['grade_distrib'].append( (row['grade'], row['count_grade'])) if ((prob_grade_distrib[curr_problem]['max_grade'] != row['max_grade']) and (prob_grade_distrib[curr_problem]['max_grade'] < row['max_grade'])): prob_grade_distrib[curr_problem]['max_grade'] = row[ 'max_grade'] else: prob_grade_distrib[curr_problem] = { 'max_grade': row['max_grade'], 'grade_distrib': [ (row['grade'], row['count_grade']), ], } # Build set of total students attempting each problem total_student_count[curr_problem] += row['count_grade'] else: # Retrieve course object down to problems course = modulestore().get_course(course_id, depth=4) # Connect to analytics data client client = Client(base_url=settings.ANALYTICS_DATA_URL, auth_token=settings.ANALYTICS_DATA_TOKEN) for section in course.get_children(): for subsection in section.get_children(): for unit in subsection.get_children(): for child in unit.get_children(): if child.location.category not in PROB_TYPE_LIST: continue problem_id = child.location problem = client.modules(course_id, problem_id) try: grade_distribution = problem.grade_distribution() except NotFoundError: grade_distribution = [] for score in grade_distribution: total_student_count[problem_id] += score['count'] if problem_id in prob_grade_distrib: if prob_grade_distrib[problem_id][ 'max_grade'] < score['max_grade']: prob_grade_distrib[problem_id][ 'max_grade'] = score['max_grade'] prob_grade_distrib[problem_id][ 'grade_distrib'].append( (score['grade'], score['count'])) else: prob_grade_distrib[problem_id] = { 'max_grade': score['max_grade'], 'grade_distrib': [ (score['grade'], score['count']), ], } return prob_grade_distrib, total_student_count