def run(self): self.remove_output_on_overwrite() courses = [] with self.input()['course_list'].open('r') as course_list_file: for line in course_list_file: course = CourseRecord.from_tsv(line) courses.append(course.course_id) client = EdxApiClient(token_type=self.api_token_type) params = dict(depth="all", requested_fields="children", all_blocks="true") counter = 0 with self.output().open('w') as output_file: for course_id in courses: # pylint: disable=not-an-iterable params['course_id'] = course_id try: # Course Blocks are returned on one page response = client.get(self.api_root_url, params=params) except HTTPError as error: # 404 errors may occur if we try to fetch the course blocks for a deleted course. # So we just log and ignore them. if error.response.status_code == 404: log.error('Error fetching API resource %s: %s', params, error) else: raise error else: parsed_response = response.json() parsed_response['course_id'] = course_id output_file.write(json.dumps(parsed_response)) output_file.write('\n') counter += 1 log.info('Wrote %d records to output file', counter)
def setUp(self): self.current_time = datetime(2016, 1, 1, 0, 0, 0, 0) self.time_offset = 0 class MockDateTime(datetime): """ A fake datetime that allows the test to control the return value from the utcnow() method. This allows us to emulate the passage of time, by simply changing `self.time_offset`. """ @classmethod def utcnow(cls): """Return the time specified by the time offset""" return self.current_time + timedelta(seconds=self.time_offset) datetime_patcher = patch('edx.analytics.tasks.util.edx_api_client.datetime', MockDateTime) datetime_patcher.start() self.addCleanup(datetime_patcher.stop) # don't actually sleep! sleep_patcher = patch('edx.analytics.tasks.util.retry.time.sleep', return_value=None) self.mock_sleep = sleep_patcher.start() self.addCleanup(sleep_patcher.stop) self.client = EdxApiClient(auth_url=FAKE_AUTH_URL, client_id=FAKE_CLIENT_ID, client_secret=FAKE_CLIENT_SECRET)
def run(self): self.remove_output_on_overwrite() client = EdxApiClient() with self.output().open('w') as output_file: short_codes = self.partner_short_codes if self.partner_short_codes else [] for partner_short_code in short_codes: params = { 'limit': self.api_page_size, 'partner': partner_short_code, 'exclude_utm': 1, } if not self.api_root_url: raise luigi.parameter.MissingParameterException( "Missing api_root_url.") url = url_path_join(self.api_root_url, 'course_runs') + '/' for response in client.paginated_get(url, params=params): parsed_response = response.json() counter = 0 for course in parsed_response.get('results', []): course['partner_short_code'] = partner_short_code output_file.write(json.dumps(course)) output_file.write('\n') counter += 1 if counter > 0: log.info('Wrote %d records to output file', counter)
def test_oauth_dogwood_from_config(self): self.prepare_for_token_request() client = EdxApiClient() client.ensure_oauth_access_token() self.assert_dogwood_token_request_body('cidfromcfg', 'secfromcfg', 'unamefromcfg', 'pwdfromcfg') self.assertEqual(client.authenticated_session.auth.token, FAKE_ACCESS_TOKEN)
def test_token_type(self): self.prepare_for_token_request() client = EdxApiClient(token_type='bearer') client.ensure_oauth_access_token() self.assert_token_request_body('cidfromcfg', 'secfromcfg', token_type='bearer') self.assertEqual(client.authenticated_session.auth.token, FAKE_ACCESS_TOKEN)
def run(self): self.remove_output_on_overwrite() client = EdxApiClient() with self.output().open('w') as output_file: for partner_short_code in self.partner_short_codes: # pylint: disable=not-an-iterable params = { 'limit': self.api_page_size, 'partner': partner_short_code } url = url_path_join(self.api_root_url, 'course_runs') + '/' for response in client.paginated_get(url, params=params): parsed_response = response.json() counter = 0 for course in parsed_response.get('results', []): course['partner_short_code'] = partner_short_code output_file.write(json.dumps(course)) output_file.write('\n') counter += 1 if counter > 0: log.info('Wrote %d records to output file', counter)
def run(self): self.remove_output_on_overwrite() client = EdxApiClient() params = { 'page_size': self.api_page_size, } def _pagination(response): """Gets the next URL from the course list API response.""" return response.get('pagination', {}).get('next') counter = 0 with self.output().open('w') as output_file: for response in client.paginated_get(self.api_root_url, params=params, pagination_key=_pagination): parsed_response = response.json() for course in parsed_response.get('results', []): output_file.write(json.dumps(course)) output_file.write('\n') counter += 1 log.info('Wrote %d records to output file', counter)
def run(self): self.remove_output_on_overwrite() client = EdxApiClient() with self.output().open('w') as output_file: short_codes = self.partner_short_codes if self.partner_short_codes else [] for partner_short_code in short_codes: params = { 'limit': self.api_page_size, 'partner': partner_short_code, 'exclude_utm': 1, } if self.partner_api_urls: url_index = short_codes.index(partner_short_code) if url_index >= self.partner_api_urls.__len__(): raise luigi.parameter.MissingParameterException( "Error! Index of the partner short code from partner_short_codes exceeds the length of " "partner_api_urls. These lists are not in sync!!!" ) api_root_url = self.partner_api_urls[url_index] elif self.api_root_url: api_root_url = self.api_root_url else: raise luigi.parameter.MissingParameterException( "Missing either a partner_api_urls or an " + "api_root_url.") url = url_path_join(api_root_url, 'courses') + '/' for response in client.paginated_get(url, params=params): parsed_response = response.json() counter = 0 for course in parsed_response.get('results', []): course['partner_short_code'] = partner_short_code output_file.write(json.dumps(course)) output_file.write('\n') counter += 1 if counter > 0: log.info('Wrote %d records to output file', counter)