def test_strip_trailing_slash(self):
        url = 'http://example.com'
        client = Client(url)
        self.assertEqual(client.base_url, url)

        url_with_slash = 'http://example.com/'
        client = Client(url_with_slash)
        self.assertEqual(client.base_url, url)
예제 #2
0
def health(_request):
    OK = 'OK'
    UNAVAILABLE = 'UNAVAILABLE'

    overall_status = analytics_api_status = database_status = UNAVAILABLE

    try:
        cursor = connection.cursor()
        cursor.execute("SELECT 1")
        cursor.fetchone()
        cursor.close()
        database_status = OK
    except DatabaseError:  # pylint: disable=catching-non-exception
        database_status = UNAVAILABLE

    try:
        client = Client(base_url=settings.DATA_API_URL, auth_token=settings.DATA_API_AUTH_TOKEN)
        if client.status.healthy:
            analytics_api_status = OK
    except ClientError as e:
        logger.exception('API is not reachable from dashboard: %s', e)
        analytics_api_status = UNAVAILABLE

    overall_status = OK if (analytics_api_status == database_status == OK) else UNAVAILABLE

    data = {
        'overall_status': overall_status,
        'detailed_status': {
            'database_connection': database_status,
            'analytics_api': analytics_api_status
        }
    }

    return HttpResponse(json.dumps(data), content_type='application/json', status=200 if overall_status == OK else 503)
    def api_client(self):

        analytics_api_client = Client(base_url=self.partner.analytics_url,
                                      auth_token=self.partner.analytics_token,
                                      timeout=self.API_TIMEOUT)

        return analytics_api_client
def _update_active_students(course_key, section_data):
    auth_token = settings.ANALYTICS_DATA_TOKEN
    base_url = settings.ANALYTICS_DATA_URL

    section_data['active_student_count'] = 'N/A'
    section_data['active_student_count_start'] = 'N/A'
    section_data['active_student_count_end'] = 'N/A'

    try:
        client = Client(base_url=base_url, auth_token=auth_token)
        course = client.courses(unicode(course_key))

        recent_activity = course.recent_activity()
        section_data['active_student_count'] = recent_activity['count']

        def format_date(value):
            return value.split('T')[0]

        start = recent_activity['interval_start']
        end = recent_activity['interval_end']

        section_data['active_student_count_start'] = format_date(start)
        section_data['active_student_count_end'] = format_date(end)

    except (ClientError, KeyError) as e:
        log.exception(e)
예제 #5
0
 def setup(self, request, *args, **kwargs):
     super().setup(request, *args, **kwargs)
     api_version = request.GET.get('v', '0')
     analytics_base_url = settings.DATA_API_URL_V1 if api_version == '1' else settings.DATA_API_URL
     self.analytics_client = Client(base_url=analytics_base_url,
                                    auth_token=settings.DATA_API_AUTH_TOKEN,
                                    timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT)
예제 #6
0
def main():
    start = time.time()
    api_client = Client(base_url=API_SERVER_URL, auth_token=API_AUTH_TOKEN, timeout=1000)
    http_client = requests.Session()

    if BASIC_AUTH_CREDENTIALS:
        http_client.auth = BASIC_AUTH_CREDENTIALS

    login(http_client)

    # Basic auth is no longer needed
    http_client.auth = None

    # Get courses
    courses = get_courses(http_client)

    # Collect the data
    reports = Queue()
    try:
        p = Pool(NUM_PROCESSES, pool_init, [reports, api_client, http_client.cookies])
        p.map(check_course, courses)
    except Exception as e:  # pylint: disable=broad-except
        logger.error('Validation failed to finish: %s', e)

    # Write the data to an external file
    write_csv(reports)
    end = time.time()

    logger.info('Finished in %d seconds.', end - start)
예제 #7
0
 def get_context_data(self, **kwargs):
     context = super(CourseView, self).get_context_data(**kwargs)
     self.client = Client(base_url=settings.DATA_API_URL,
                          auth_token=settings.DATA_API_AUTH_TOKEN,
                          timeout=settings.LMS_DEFAULT_TIMEOUT)
     self.course = self.client.courses(self.course_id)
     return context
예제 #8
0
    def setUp(self):
        super().setUp()

        api_url = API_SERVER_URL
        auth_token = API_AUTH_TOKEN
        self.analytics_api_client = Client(api_url,
                                           auth_token=auth_token,
                                           timeout=10)
예제 #9
0
 def __init__(self,
              course_id,
              timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT):
     self.client = Client(base_url=settings.DATA_API_URL,
                          auth_token=settings.DATA_API_AUTH_TOKEN,
                          timeout=timeout)
     self.course_id = course_id
     self.course = self.client.courses(self.course_id)
예제 #10
0
 def test_no_summaries(self):
     cache.clear()  # previous test has course_ids=None case cached
     presenter = CourseSummariesPresenter(Client('base_url'))
     with mock.patch(
             'analyticsclient.course_summaries.CourseSummaries.course_summaries',
             mock.Mock(return_value=[])):
         summaries, last_updated = presenter.get_course_summaries()
         self.assertListEqual(summaries, [])
         self.assertIsNone(last_updated)
    def test_get_programs(self, program_ids, course_ids):
        ''''Test programs filtered from API response.'''
        presenter = ProgramsPresenter(Client('base_url'))

        with mock.patch('analyticsclient.programs.Programs.programs',
                        mock.Mock(return_value=self.mock_api_response)):
            actual_programs = presenter.get_programs(program_ids=program_ids, course_ids=course_ids)
            self.assertListEqual(actual_programs, self.get_expected_programs(program_ids=program_ids,
                                                                             course_ids=course_ids))
 def __init__(self, course_id, http_cookies=None):
     self.course_id = course_id
     self.analytics_api_client = Client(base_url=API_SERVER_URL,
                                        auth_token=API_AUTH_TOKEN,
                                        timeout=1000)
     self.course_api_client = CourseStructureApiClient(
         COURSE_API_URL, COURSE_API_KEY, 5)
     self.http_client = requests.Session()
     self.http_client.cookies = http_cookies
예제 #13
0
    def test_failed_authentication(self):
        client = Client(base_url=self.api_url, auth_token='atoken')
        httpretty.register_uri(httpretty.GET,
                               self.test_url,
                               body='',
                               status=401)

        self.assertEqual(client.has_resource(self.test_endpoint), False)
        self.assertEqual(httpretty.last_request().headers['Authorization'],
                         'Token atoken')
예제 #14
0
 def test_get_course_summary_metrics(self):
     presenter = CourseSummariesPresenter(Client('base_url'))
     metrics = presenter.get_course_summary_metrics(
         self._PRESENTER_SUMMARIES.values())
     expected = {
         'total_enrollment': 5111,
         'current_enrollment': 3888,
         'enrollment_change_7_days': 4,
         'verified_enrollment': 13,
         'masters_enrollment': 1111,
     }
     self.assertEqual(metrics, expected)
예제 #15
0
def health(_request):
    if newrelic:  # pragma: no cover
        newrelic.agent.ignore_transaction()
    overall_status = analytics_api_status = database_status = UNAVAILABLE

    try:
        cursor = connection.cursor()
        cursor.execute("SELECT 1")
        cursor.fetchone()
        cursor.close()
        database_status = OK
    except DatabaseError as e:
        logger.exception('Insights database is not reachable: %s', e)
        database_status = UNAVAILABLE

    try:
        client = Client(base_url=settings.DATA_API_URL,
                        auth_token=settings.DATA_API_AUTH_TOKEN,
                        timeout=0.35)
        # Note: client.status.healthy sends a request to the health endpoint on
        # the Analytics API.  The request may throw a TimeoutError.  Currently,
        # other exceptions are caught by the client.status.healthy method
        # itself, which will return False in those cases.
        analytics_api_healthy = client.status.healthy
    except TimeoutError as e:
        logger.exception(
            'Analytics API health check timed out from dashboard: %s', e)
        analytics_api_status = UNAVAILABLE
    else:
        if analytics_api_healthy:
            analytics_api_status = OK
        else:
            logger.error('Analytics API health check failed from dashboard')
            analytics_api_status = UNAVAILABLE

    overall_status = OK if (
        analytics_api_status == database_status == OK) else UNAVAILABLE

    data = {
        'overall_status': overall_status,
        'detailed_status': {
            'database_connection': database_status,
            'analytics_api': analytics_api_status
        }
    }

    return HttpResponse(json.dumps(data),
                        content_type='application/json',
                        status=200 if overall_status == OK else 503)
예제 #16
0
def get_sequential_open_distrib(course_id, enrollment):
    """
    Returns the number of students that opened each subsection/sequential of the course

    `course_id` the course ID for the course interested in

    `enrollment` the number of students enrolled in this course.

    Outputs a dict mapping the 'module_id' to the number of students that have opened that subsection/sequential.
    """
    sequential_open_distrib = {}

    non_student_list = get_non_student_list(course_id)

    if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL:
        # Aggregate query on studentmodule table for "opening a subsection" data
        queryset = models.StudentModule.objects.filter(
            course_id__exact=course_id,
            module_type__exact='sequential',
        ).exclude(student_id__in=non_student_list).values(
            'module_state_key').annotate(
                count_sequential=Count('module_state_key'))

        for row in queryset:
            module_id = course_id.make_usage_key_from_deprecated_string(
                row['module_state_key'])
            sequential_open_distrib[module_id] = row['count_sequential']
    else:
        # Retrieve course object down to subsection
        course = modulestore().get_course(course_id, depth=2)

        # Connect to analytics data client
        client = Client(base_url=settings.ANALYTICS_DATA_URL,
                        auth_token=settings.ANALYTICS_DATA_TOKEN)

        for section in course.get_children():
            for subsection in section.get_children():
                module = client.modules(course_id, subsection.location)

                try:
                    sequential_open = module.sequential_open_distribution()
                except NotFoundError:
                    pass
                else:
                    sequential_open_distrib[
                        subsection.location] = sequential_open[0]['count']

    return sequential_open_distrib
예제 #17
0
    def test_get_summaries(self, input_course_ids, ouptut_course_ids):
        presenter = CourseSummariesPresenter(Client('base_url'))
        if input_course_ids:
            mock_api_response = [
                self._API_SUMMARIES[course_id]
                for course_id in input_course_ids
            ]
        else:
            mock_api_response = list(self._API_SUMMARIES.values())
        expected_summaries = [
            self._PRESENTER_SUMMARIES[course_id]
            for course_id in ouptut_course_ids
        ]

        with mock.patch(
                'analyticsclient.course_summaries.CourseSummaries.course_summaries',
                mock.Mock(return_value=mock_api_response)):
            actual_summaries, last_updated = presenter.get_course_summaries(
                course_ids=input_course_ids)
            for actual, expected in zip(actual_summaries, expected_summaries):
                self.assertCountEqual(actual, expected)
            self.assertEqual(last_updated, utils.CREATED_DATETIME)
 def __init__(self, course_id, timeout=5):
     self.client = Client(base_url=settings.DATA_API_URL,
                          auth_token=settings.DATA_API_AUTH_TOKEN,
                          timeout=timeout)
     self.course_id = course_id
     self.course = self.client.courses(self.course_id)
예제 #19
0
 def __init__(self):
     View.__init__(self)
     self.analytics_client = Client(base_url=settings.ANALYTICS_API_URL,
                                    auth_token=settings.ANALYTICS_API_KEY)
예제 #20
0
 def setUp(self):
     """Configure Client."""
     self.api_url = 'http://localhost:9999/api/v1'
     self.client = Client(self.api_url)
 def test_date_format(self):
     self.assertEqual(Client.DATE_FORMAT, '%Y-%m-%d')
     self.assertEqual(Client('').DATE_FORMAT, '%Y-%m-%d')
예제 #22
0
def get_problem_set_grade_distrib(course_id, problem_set, enrollment):
    """
    Returns the grade distribution for the problems specified in `problem_set`.

    `course_id` the course ID for the course interested in

    `problem_set` an array of UsageKeys representing problem module_id's.

    `enrollment` the number of students enrolled in this course.

    Requests from the database the a count of each grade for each problem in the `problem_set`.

    Returns a dict, where the key is the problem 'module_id' and the value is a dict with two parts:
      'max_grade' - the maximum grade possible for the course
      'grade_distrib' - array of tuples (`grade`,`count`) ordered by `grade`
    """

    non_student_list = get_non_student_list(course_id)

    prob_grade_distrib = {}

    if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL:
        # Aggregate query on studentmodule table for grade data for set of problems in course
        queryset = models.StudentModule.objects.filter(
            course_id__exact=course_id,
            grade__isnull=False,
            module_type__in=PROB_TYPE_LIST,
            module_state_key__in=problem_set,
        ).exclude(student_id__in=non_student_list).values(
            'module_state_key',
            'grade',
            'max_grade',
        ).annotate(count_grade=Count('grade')).order_by(
            'module_state_key', 'grade')

        # Loop through resultset building data for each problem
        for row in queryset:
            problem_id = course_id.make_usage_key_from_deprecated_string(
                row['module_state_key'])
            if problem_id not in prob_grade_distrib:
                prob_grade_distrib[problem_id] = {
                    'max_grade': 0,
                    'grade_distrib': [],
                }

            curr_grade_distrib = prob_grade_distrib[problem_id]
            curr_grade_distrib['grade_distrib'].append(
                (row['grade'], row['count_grade']))

            if curr_grade_distrib['max_grade'] < row['max_grade']:
                curr_grade_distrib['max_grade'] = row['max_grade']
    else:
        # Connect to analytics data client
        client = Client(base_url=settings.ANALYTICS_DATA_URL,
                        auth_token=settings.ANALYTICS_DATA_TOKEN)

        for problem in problem_set:
            module = client.modules(course_id, problem)

            try:
                grade_distribution = module.grade_distribution()
            except NotFoundError:
                grade_distribution = []

            for score in grade_distribution:
                if problem in prob_grade_distrib:
                    if prob_grade_distrib[problem]['max_grade'] < score[
                            'max_grade']:
                        prob_grade_distrib[problem]['max_grade'] = score[
                            'max_grade']

                    prob_grade_distrib[problem]['grade_distrib'].append(
                        (score['grade'], score['count']))
                else:
                    prob_grade_distrib[problem] = {
                        'max_grade': score['max_grade'],
                        'grade_distrib': [(score['grade'], score['count'])],
                    }

    return prob_grade_distrib
예제 #23
0
 def __init__(self, timeout=settings.ANALYTICS_API_DEFAULT_TIMEOUT):
     self.client = Client(base_url=settings.DATA_API_URL_SNAIL2, auth_token=settings.DATA_API_AUTH_TOKEN_SNAIL, timeout=timeout)
예제 #24
0
def get_problem_grade_distribution(course_id, enrollment):
    """
    Returns the grade distribution per problem for the course

    `course_id` the course ID for the course interested in

    `enrollment` the number of students enrolled in this course.

    Output is 2 dicts:
      'prob-grade_distrib' where the key is the problem 'module_id' and the value is a dict with:
        'max_grade' - max grade for this problem
        'grade_distrib' - array of tuples (`grade`,`count`).
      'total_student_count' where the key is problem 'module_id' and the value is number of students
        attempting the problem
    """
    non_student_list = get_non_student_list(course_id)

    prob_grade_distrib = {}
    total_student_count = defaultdict(int)

    if enrollment <= settings.MAX_ENROLLEES_FOR_METRICS_USING_DB or not settings.ANALYTICS_DATA_URL:
        # Aggregate query on studentmodule table for grade data for all problems in course
        queryset = models.StudentModule.objects.filter(
            course_id__exact=course_id,
            grade__isnull=False,
            module_type__in=PROB_TYPE_LIST,
        ).exclude(student_id__in=non_student_list).values(
            'module_state_key', 'grade',
            'max_grade').annotate(count_grade=Count('grade'))

        # Loop through resultset building data for each problem
        for row in queryset:
            curr_problem = course_id.make_usage_key_from_deprecated_string(
                row['module_state_key'])

            # Build set of grade distributions for each problem that has student responses
            if curr_problem in prob_grade_distrib:
                prob_grade_distrib[curr_problem]['grade_distrib'].append(
                    (row['grade'], row['count_grade']))

                if ((prob_grade_distrib[curr_problem]['max_grade'] !=
                     row['max_grade'])
                        and (prob_grade_distrib[curr_problem]['max_grade'] <
                             row['max_grade'])):
                    prob_grade_distrib[curr_problem]['max_grade'] = row[
                        'max_grade']

            else:
                prob_grade_distrib[curr_problem] = {
                    'max_grade': row['max_grade'],
                    'grade_distrib': [
                        (row['grade'], row['count_grade']),
                    ],
                }

            # Build set of total students attempting each problem
            total_student_count[curr_problem] += row['count_grade']
    else:
        # Retrieve course object down to problems
        course = modulestore().get_course(course_id, depth=4)

        # Connect to analytics data client
        client = Client(base_url=settings.ANALYTICS_DATA_URL,
                        auth_token=settings.ANALYTICS_DATA_TOKEN)

        for section in course.get_children():
            for subsection in section.get_children():
                for unit in subsection.get_children():
                    for child in unit.get_children():
                        if child.location.category not in PROB_TYPE_LIST:
                            continue

                        problem_id = child.location
                        problem = client.modules(course_id, problem_id)

                        try:
                            grade_distribution = problem.grade_distribution()
                        except NotFoundError:
                            grade_distribution = []

                        for score in grade_distribution:
                            total_student_count[problem_id] += score['count']

                            if problem_id in prob_grade_distrib:
                                if prob_grade_distrib[problem_id][
                                        'max_grade'] < score['max_grade']:
                                    prob_grade_distrib[problem_id][
                                        'max_grade'] = score['max_grade']

                                prob_grade_distrib[problem_id][
                                    'grade_distrib'].append(
                                        (score['grade'], score['count']))
                            else:
                                prob_grade_distrib[problem_id] = {
                                    'max_grade':
                                    score['max_grade'],
                                    'grade_distrib': [
                                        (score['grade'], score['count']),
                                    ],
                                }

    return prob_grade_distrib, total_student_count