def run(self):
self.remove_output_on_overwrite()
courses = []
with self.input()['course_list'].open('r') as course_list_file:
for line in course_list_file:
course = CourseRecord.from_tsv(line)
courses.append(course.course_id)
client = EdxApiClient(token_type=self.api_token_type)
params = dict(depth="all", requested_fields="children", all_blocks="true")
counter = 0
with self.output().open('w') as output_file:
for course_id in courses: # pylint: disable=not-an-iterable
params['course_id'] = course_id
try:
# Course Blocks are returned on one page
response = client.get(self.api_root_url, params=params)
except HTTPError as error:
# 404 errors may occur if we try to fetch the course blocks for a deleted course.
# So we just log and ignore them.
if error.response.status_code == 404:
log.error('Error fetching API resource %s: %s', params, error)
else:
raise error
else:
parsed_response = response.json()
parsed_response['course_id'] = course_id
output_file.write(json.dumps(parsed_response))
output_file.write('\n')
counter += 1
log.info('Wrote %d records to output file', counter)
def setUp(self):
self.current_time = datetime(2016, 1, 1, 0, 0, 0, 0)
self.time_offset = 0
class MockDateTime(datetime):
"""
A fake datetime that allows the test to control the return value from the utcnow() method.
This allows us to emulate the passage of time, by simply changing `self.time_offset`.
"""
@classmethod
def utcnow(cls):
"""Return the time specified by the time offset"""
return self.current_time + timedelta(seconds=self.time_offset)
datetime_patcher = patch('edx.analytics.tasks.util.edx_api_client.datetime', MockDateTime)
datetime_patcher.start()
self.addCleanup(datetime_patcher.stop)
# don't actually sleep!
sleep_patcher = patch('edx.analytics.tasks.util.retry.time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.client = EdxApiClient(auth_url=FAKE_AUTH_URL, client_id=FAKE_CLIENT_ID, client_secret=FAKE_CLIENT_SECRET)
def run(self):
self.remove_output_on_overwrite()
client = EdxApiClient()
with self.output().open('w') as output_file:
short_codes = self.partner_short_codes if self.partner_short_codes else []
for partner_short_code in short_codes:
params = {
'limit': self.api_page_size,
'partner': partner_short_code,
'exclude_utm': 1,
}
if not self.api_root_url:
raise luigi.parameter.MissingParameterException(
"Missing api_root_url.")
url = url_path_join(self.api_root_url, 'course_runs') + '/'
for response in client.paginated_get(url, params=params):
parsed_response = response.json()
counter = 0
for course in parsed_response.get('results', []):
course['partner_short_code'] = partner_short_code
output_file.write(json.dumps(course))
output_file.write('\n')
counter += 1
if counter > 0:
log.info('Wrote %d records to output file', counter)
def test_oauth_dogwood_from_config(self):
self.prepare_for_token_request()
client = EdxApiClient()
client.ensure_oauth_access_token()
self.assert_dogwood_token_request_body('cidfromcfg', 'secfromcfg', 'unamefromcfg', 'pwdfromcfg')
self.assertEqual(client.authenticated_session.auth.token, FAKE_ACCESS_TOKEN)
def test_token_type(self):
self.prepare_for_token_request()
client = EdxApiClient(token_type='bearer')
client.ensure_oauth_access_token()
self.assert_token_request_body('cidfromcfg', 'secfromcfg', token_type='bearer')
self.assertEqual(client.authenticated_session.auth.token, FAKE_ACCESS_TOKEN)
def run(self):
self.remove_output_on_overwrite()
client = EdxApiClient()
with self.output().open('w') as output_file:
for partner_short_code in self.partner_short_codes: # pylint: disable=not-an-iterable
params = {
'limit': self.api_page_size,
'partner': partner_short_code
}
url = url_path_join(self.api_root_url, 'course_runs') + '/'
for response in client.paginated_get(url, params=params):
parsed_response = response.json()
counter = 0
for course in parsed_response.get('results', []):
course['partner_short_code'] = partner_short_code
output_file.write(json.dumps(course))
output_file.write('\n')
counter += 1
if counter > 0:
log.info('Wrote %d records to output file', counter)
def run(self):
self.remove_output_on_overwrite()
client = EdxApiClient()
params = {
'page_size': self.api_page_size,
}
def _pagination(response):
"""Gets the next URL from the course list API response."""
return response.get('pagination', {}).get('next')
counter = 0
with self.output().open('w') as output_file:
for response in client.paginated_get(self.api_root_url, params=params, pagination_key=_pagination):
parsed_response = response.json()
for course in parsed_response.get('results', []):
output_file.write(json.dumps(course))
output_file.write('\n')
counter += 1
log.info('Wrote %d records to output file', counter)
def run(self):
self.remove_output_on_overwrite()
client = EdxApiClient()
with self.output().open('w') as output_file:
short_codes = self.partner_short_codes if self.partner_short_codes else []
for partner_short_code in short_codes:
params = {
'limit': self.api_page_size,
'partner': partner_short_code,
'exclude_utm': 1,
}
if self.partner_api_urls:
url_index = short_codes.index(partner_short_code)
if url_index >= self.partner_api_urls.__len__():
raise luigi.parameter.MissingParameterException(
"Error! Index of the partner short code from partner_short_codes exceeds the length of "
"partner_api_urls. These lists are not in sync!!!"
)
api_root_url = self.partner_api_urls[url_index]
elif self.api_root_url:
api_root_url = self.api_root_url
else:
raise luigi.parameter.MissingParameterException(
"Missing either a partner_api_urls or an " +
"api_root_url.")
url = url_path_join(api_root_url, 'courses') + '/'
for response in client.paginated_get(url, params=params):
parsed_response = response.json()
counter = 0
for course in parsed_response.get('results', []):
course['partner_short_code'] = partner_short_code
output_file.write(json.dumps(course))
output_file.write('\n')
counter += 1
if counter > 0:
log.info('Wrote %d records to output file', counter)
