def access_token(self):
""" Returns an access token for this site's service user.
The access token is retrieved using the current site's OAuth credentials and the client credentials grant.
The token is cached for the lifetime of the token, as specified by the OAuth provider's response. The token
type is JWT.
Returns:
str: JWT access token
"""
key = 'siteconfiguration_access_token_{}'.format(self.id)
access_token = cache.get(key)
# pylint: disable=unsubscriptable-object
if not access_token:
url = '{root}/access_token'.format(root=self.oauth2_provider_url)
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token(
url,
self.oauth_settings['SOCIAL_AUTH_EDX_OIDC_KEY'],
self.oauth_settings['SOCIAL_AUTH_EDX_OIDC_SECRET'],
token_type='jwt'
)
expires = (expiration_datetime - datetime.datetime.utcnow()).seconds
cache.set(key, access_token, expires)
return access_token
def access_token(self):
""" Returns an access token for this site's service user.
The access token is retrieved using the current site's OAuth credentials and the client credentials grant.
The token is cached for the lifetime of the token, as specified by the OAuth provider's response. The token
type is JWT.
Returns:
str: JWT access token
"""
key = f'siteconfiguration_access_token_{self.id}'
access_token = cache.get(key)
if not access_token:
url = f'{self.oauth2_provider_url}/access_token'
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token(
url,
self.oauth2_client_id,
self.oauth2_client_secret,
token_type='jwt'
)
expires = (expiration_datetime - datetime.datetime.utcnow()).seconds
cache.set(key, access_token, expires)
return access_token
def access_token(self):
"""
Returns the access token for this service.
We don't use a cached_property decorator because we aren't sure how to
set custom expiration dates for those.
Returns:
str: JWT access token
"""
key = 'oauth2_access_token'
access_token = cache.get(key)
if not access_token:
url = '{root}/access_token'.format(root=self.oauth2_provider_url)
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token(
url,
self.oauth2_client_id,
self.oauth2_client_secret,
token_type='jwt')
expires = (expiration_datetime -
datetime.datetime.utcnow()).seconds
cache.set(key, access_token, expires)
return access_token
def access_token(self):
""" Returns an access token for this site's service user.
The access token is retrieved using the current site's OAuth credentials and the client credentials grant.
The token is cached for the lifetime of the token, as specified by the OAuth provider's response. The token
type is JWT.
Returns:
str: JWT access token
"""
key = 'siteconfiguration_access_token_{}'.format(self.id)
access_token_cached_response = TieredCache.get_cached_response(key)
if access_token_cached_response.is_found:
return access_token_cached_response.value
url = '{root}/access_token'.format(root=self.oauth2_provider_url)
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token(
url,
self.oauth_settings['BACKEND_SERVICE_EDX_OAUTH2_KEY'], # pylint: disable=unsubscriptable-object
self.oauth_settings['BACKEND_SERVICE_EDX_OAUTH2_SECRET'], # pylint: disable=unsubscriptable-object
token_type='jwt')
expires = (expiration_datetime - datetime.datetime.utcnow()).seconds
TieredCache.set_all_tiers(key, access_token, expires)
return access_token
def get_access_token():
""" Returns an access token and expiration date from the OAuth provider.
Returns:
(str, datetime)
"""
return EdxRestApiClient.get_oauth_access_token(
OAUTH_ACCESS_TOKEN_URL, OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, token_type='jwt'
)
def handle(self, *args, **options):
# For each partner defined...
partners = Partner.objects.all()
# If a specific partner was indicated, filter down the set
partner_code = options.get('partner_code')
if partner_code:
partners = partners.filter(short_code=partner_code)
if not partners:
raise CommandError('No partners available!')
for partner in partners:
access_token = options.get('access_token')
token_type = options.get('token_type')
if access_token and not token_type:
raise CommandError('The token_type must be specified when passing in an access token!')
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
token_type = 'JWT'
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=partner.oidc_url_root.strip('/')),
partner.oidc_key,
partner.oidc_secret,
token_type=token_type
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
loaders = []
if partner.organizations_api_url:
loaders.append(OrganizationsApiDataLoader)
if partner.courses_api_url:
loaders.append(CoursesApiDataLoader)
if partner.ecommerce_api_url:
loaders.append(EcommerceApiDataLoader)
if partner.marketing_site_api_url:
loaders.append(DrupalApiDataLoader)
if partner.programs_api_url:
loaders.append(ProgramsApiDataLoader)
if loaders:
for loader_class in loaders:
try:
loader_class(partner, access_token, token_type).ingest()
except Exception: # pylint: disable=broad-except
logger.exception('%s failed!', loader_class.__name__)
def get_access_token():
""" Returns an access token and expiration date from the OAuth
provider.
Returns:
(str, datetime):Tuple containing access token and expiration date.
"""
return EdxRestApiClient.get_oauth_access_token(
OAUTH_ACCESS_TOKEN_URL,
OAUTH_CLIENT_ID,
OAUTH_CLIENT_SECRET,
token_type='jwt'
)
def connect(self):
"""
Connect to the REST API, authenticating with an access token retrieved with our client credentials.
"""
access_token, expires_at = EdxRestApiClient.get_oauth_access_token(
self.LMS_OAUTH_HOST + '/oauth2/access_token', self.client_id,
self.client_secret, 'jwt')
self.client = EdxRestApiClient(
self.API_BASE_URL,
append_slash=self.APPEND_SLASH,
jwt=access_token,
)
self.expires_at = expires_at
def __init__(self):
jwt, __ = EdxRestApiClient.get_oauth_access_token(
f'{OIDC_URL}/access_token',
OIDC_KEY,
OIDC_SECRET,
token_type='jwt'
)
self.client = EdxRestApiClient(DISCOVERY_API_URL, jwt=jwt)
self.deadline_empty_with_end = []
self.deadline_empty_without_end = []
self.deadline_after_end = []
def test_get_client_credential_access_token_success(self):
""" Test that the get access token method handles 200 responses and returns the access token. """
code = 200
body = {"access_token": "my-token", "expires_in": 1000}
now = datetime.datetime.utcnow()
expected_return = ("my-token", now + datetime.timedelta(seconds=1000))
with freeze_time(now):
self._mock_auth_api(URL, code, body=body)
self.assertEqual(
EdxRestApiClient.get_oauth_access_token(
URL, "client_id", "client_secret"), expected_return)
def __init__(self, oauth_access_token_url, oauth_key, oauth_secret, api_url_root):
try:
self.access_token, expires = EdxRestApiClient.get_oauth_access_token(
oauth_access_token_url,
oauth_key,
oauth_secret, token_type='jwt'
)
except Exception:
logger.exception('No access token acquired through client_credential flow.')
raise
logger.info('Retrieved access token.')
self.api_client = EdxRestApiClient(api_url_root, jwt=self.access_token)
def test_get_client_credential_access_token_success(self):
""" Test that the get access token method handles 200 responses and returns the access token. """
code = 200
body = {"access_token": "my-token", "expires_in": 1000}
now = datetime.datetime.utcnow()
expected_return = ("my-token", now + datetime.timedelta(seconds=1000))
with freeze_time(now):
self._mock_auth_api(URL, code, body=body)
self.assertEqual(
EdxRestApiClient.get_oauth_access_token(URL, "client_id", "client_secret"),
expected_return
)
def get_access_token(self):
"""
Returns an access token and expiration date from the OAuth provider:
(str, datetime)
"""
logger.info('\nFetching access token for site...')
oauth2_provider_url = self.site.oauth_settings.get(
'BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL')
key = self.site.oauth_settings.get('BACKEND_SERVICE_EDX_OAUTH2_KEY')
secret = self.site.oauth_settings.get(
'BACKEND_SERVICE_EDX_OAUTH2_SECRET')
oauth_access_token_url = oauth2_provider_url + '/access_token/'
return EdxRestApiClient.get_oauth_access_token(oauth_access_token_url,
key,
secret,
token_type='jwt')
def _get_access_token(args):
access_token = args.access_token
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
try:
access_token, expires = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=args.oauth_host),
args.oauth_key,
args.oauth_secret, token_type='jwt')
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
logger.info('Token retrieved: %s', access_token)
return access_token
def get_access_token(oauth_base_url, client_id, client_secret):
"""
Returns an access token and expiration date from the OAuth provider.
Returns:
(str, datetime)
"""
try:
return EdxRestApiClient.get_oauth_access_token(
oauth_base_url + OAUTH_ACCESS_TOKEN_URL,
client_id,
client_secret,
token_type='jwt')
except HttpClientError as err:
LOG.error("API Error: {}".format(err.content))
raise
def handle(self, *args, **options):
# We only want to invalidate the API response cache once data loading
# completes. Disconnecting the api_change_receiver function from post_save
# and post_delete signals prevents model changes during data loading from
# repeatedly invalidating the cache.
for model in apps.get_app_config('course_metadata').get_models():
for signal in (post_save, post_delete):
signal.disconnect(receiver=api_change_receiver, sender=model)
# Disable the post save as we do NOT want to publish these courses to Studio
post_save.disconnect(receiver=create_course_run_in_studio_receiver,
sender=PublisherCourseRun)
config = DrupalLoaderConfig.get_solo()
token_type = 'JWT'
partner = Partner.objects.get(short_code=config.partner_code)
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(
root=partner.oidc_url_root.strip('/')),
partner.oidc_key,
partner.oidc_secret,
token_type=token_type)
except Exception:
logger.exception(
'No access token acquired through client_credential flow.')
raise
username = jwt.decode(access_token, verify=False)['preferred_username']
kwargs = {'username': username} if username else {}
logger.info('Loading Drupal data for %s with partner codes\n%s',
config.partner_code,
config.course_run_ids.replace(',', '\n'))
execute_loader(
DrupalCourseMarketingSiteDataLoader,
partner,
partner.marketing_site_url_root,
access_token,
token_type,
1, # Make this a constant of 1 for no concurrency
False,
set(config.course_run_ids.split(',')),
config.load_unpublished_course_runs,
**kwargs)
post_save.connect(receiver=create_course_run_in_studio_receiver,
sender=PublisherCourseRun)
def get_access_token(oauth_base_url, client_id, client_secret):
"""
Returns an access token and expiration date from the OAuth provider.
Returns:
(str, datetime)
"""
try:
return EdxRestApiClient.get_oauth_access_token(
oauth_base_url + OAUTH_ACCESS_TOKEN_URL, client_id, client_secret, token_type='jwt'
)
except HttpClientError as err:
LOG.error("API Error: {} with status code: {} fetching access token for client: {}".format(
err.content,
err.response.status_code, # pylint: disable=no-member
client_id,
))
raise
def __init__(self):
"""
Create an LMS API client, authenticated with the OAuth2 token for client in settings
"""
# session = Session()
# # session.headers = {"X-Edx-Api-Key": settings.OPENEDX_EDX_API_KEY}
token = EdxRestApiClient.get_oauth_access_token(
url="{}{}".format(settings.OPENEDX_PLATFORM_URI,
constants.OPENEDX_OAUTH2_TOKEN_URL),
client_id=settings.OPENEDX_OAUTH2_CLIENT_ID,
client_secret=settings.OPENEDX_OAUTH2_CLIENT_SECRET)
self.cache = False
if settings.LMS_API_USE_MEMCACHED:
self.cache = memcache.Client([settings.MEMCACHED_ADDRESS], debug=0)
self.client = EdxRestApiClient(self.API_BASE_URL,
append_slash=self.APPEND_SLASH,
username="******",
oauth_access_token=token[0])
def access_token(self):
""" Returns an access token for this site's service user.
The access token is retrieved using the current site's OAuth credentials and the client credentials grant.
The token is cached for the lifetime of the token, as specified by the OAuth provider's response. The token
type is JWT.
Returns:
str: JWT access token
"""
url = '{root}/access_token'.format(root=self.oauth2_provider_url)
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token( # pylint: disable=unused-variable
url,
self.oauth_settings['SOCIAL_AUTH_EDX_OIDC_KEY'],
self.oauth_settings['SOCIAL_AUTH_EDX_OIDC_SECRET'],
token_type='jwt')
return access_token
def __init__(self, access_token, oauth_host, oauth_key, oauth_secret, api_url_root):
self.access_token = access_token
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
try:
self.access_token, expires = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=oauth_host),
oauth_key,
oauth_secret, token_type='jwt'
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
logger.info('Token retrieved: %s', access_token)
self.api_client = EdxRestApiClient(api_url_root, jwt=self.access_token)
self._programs_dictionary = {}
def access_token(self):
""" Returns an access token for this site's service user.
Returns:
str: JWT access token
"""
key = 'partner_access_token_{}'.format(self.id)
access_token = cache.get(key)
if not access_token:
url = '{root}/access_token'.format(root=self.oidc_url_root)
access_token, expiration_datetime = EdxRestApiClient.get_oauth_access_token(
url, self.oidc_key, self.oidc_secret, token_type='jwt')
expires = (expiration_datetime -
datetime.datetime.utcnow()).seconds
cache.set(key, access_token, expires)
return access_token
def handle(self, *args, **options):
access_token = options.get('access_token')
commit = options.get('commit')
if access_token is None:
try:
access_token_url = '{}/access_token'.format(
settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT.strip('/'))
client_id = settings.SOCIAL_AUTH_EDX_OIDC_KEY
client_secret = settings.SOCIAL_AUTH_EDX_OIDC_SECRET
access_token, __ = EdxRestApiClient.get_oauth_access_token(
access_token_url, client_id, client_secret)
except: # pylint: disable=bare-except
logger.exception(
'Unable to exchange client credentials grant for an access token.'
)
return
self.client = EdxRestApiClient(settings.ORGANIZATIONS_API_URL_ROOT,
oauth_access_token=access_token)
logger.info('Retrieving organization data from %s.',
settings.ORGANIZATIONS_API_URL_ROOT)
try:
with transaction.atomic():
self._get_data()
logger.info(
'Retrieved %d organizations from %s, %d of which were new.',
self.org_count, settings.ORGANIZATIONS_API_URL_ROOT,
self.new_org_count)
if not commit:
raise ForcedRollback(
'No data has been saved. To save data, pass the -c or --commit flags.'
)
except ForcedRollback as e:
logger.info(e)
def handle(self, *args, **options):
access_token = options.get('access_token')
commit = options.get('commit')
if access_token is None:
try:
access_token_url = '{}/access_token'.format(
settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT.strip('/')
)
client_id = settings.SOCIAL_AUTH_EDX_OIDC_KEY
client_secret = settings.SOCIAL_AUTH_EDX_OIDC_SECRET
access_token, __ = EdxRestApiClient.get_oauth_access_token(
access_token_url,
client_id,
client_secret
)
except: # pylint: disable=bare-except
logger.exception('Unable to exchange client credentials grant for an access token.')
return
self.client = EdxRestApiClient(settings.ORGANIZATIONS_API_URL_ROOT, oauth_access_token=access_token)
logger.info('Retrieving organization data from %s.', settings.ORGANIZATIONS_API_URL_ROOT)
try:
with transaction.atomic():
self._get_data()
logger.info(
'Retrieved %d organizations from %s, %d of which were new.',
self.org_count,
settings.ORGANIZATIONS_API_URL_ROOT,
self.new_org_count
)
if not commit:
raise ForcedRollback('No data has been saved. To save data, pass the -c or --commit flags.')
except ForcedRollback as e:
logger.info(e)
def get_course_catalog_api_client(site):
"""
Returns an API client to access the Course Catalog service.
Arguments:
site (Site): The site for which to retrieve settings.
Returns:
EdxRestApiClient: The client to access the Course Catalog service.
"""
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=get_oauth2_provider_url()),
site.siteconfiguration.oauth_settings['SOCIAL_AUTH_EDX_OIDC_KEY'],
site.siteconfiguration.oauth_settings['SOCIAL_AUTH_EDX_OIDC_SECRET'],
token_type='jwt'
)
course_catalog_client = EdxRestApiClient(
settings.COURSE_CATALOG_API_URL,
jwt=access_token
)
return course_catalog_client
def __init__(self, access_token, oauth_host, oauth_key, oauth_secret,
api_url_root):
self.access_token = access_token
if not access_token:
logger.info(
'No access token provided. Retrieving access token using client_credential flow...'
)
try:
self.access_token, expires = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=oauth_host),
oauth_key,
oauth_secret,
token_type='jwt')
except Exception:
logger.exception(
'No access token provided or acquired through client_credential flow.'
)
raise
logger.info('Retrieved access token.')
self.api_client = EdxRestApiClient(api_url_root, jwt=self.access_token)
self._programs_dictionary = {}
def process_load(args, sc, lms_url, test):
""" Process load command
:param sc: Sailthru client
:return:
"""
# Try to get edX access token if not supplied
access_token = args.access_token
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=args.oauth_host),
args.oauth_key,
args.oauth_seret
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
# logger.info('Token retrieved: %s', access_token)
# use programs api to build table of course runs that are part of xseries
series_table = load_series_table()
client = EdxRestApiClient(args.content_api_url, oauth_access_token=access_token)
count = None
page = 1
course_runs = 0
# read the courses and create a Sailthru content item for each course_run within the course
while page:
# get a page of courses
response = client.courses().get(limit=20, offset=(page-1)*20)
count = response['count']
results = response['results']
if response['next']:
page += 1
else:
page = None
for course in results:
for course_run in course['course_runs']:
sailthru_content = create_sailthru_content(course, course_run, series_table, lms_url)
if sailthru_content:
course_runs += 1
if not test:
response = sc.api_post('content', sailthru_content)
if not response.is_ok():
logger.error("Error code %d connecting to Sailthru content api: %s",
response.json['error'],
response.json['errormsg'])
return
logger.info("Course: %s, Course_run: %s saved in Sailthru.", course['key'], course_run['key'])
else:
logger.info(sailthru_content)
logger.info('Retrieved %d courses.', count)
logger.info('Saved %d course runs in Sailthru.', course_runs)
def handle(self, *args, **options):
# For each partner defined...
partners = Partner.objects.all()
# If a specific partner was indicated, filter down the set
partner_code = options.get('partner_code')
if partner_code:
partners = partners.filter(short_code=partner_code)
if not partners:
raise CommandError('No partners available!')
token_type = 'JWT'
for partner in partners:
logger.info('Retrieving access token for partner [{}]'.format(
partner_code))
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(
root=partner.oidc_url_root.strip('/')),
partner.oidc_key,
partner.oidc_secret,
token_type=token_type)
except Exception:
logger.exception(
'No access token acquired through client_credential flow.')
raise
username = jwt.decode(access_token,
verify=False)['preferred_username']
kwargs = {'username': username} if username else {}
# The Linux kernel implements copy-on-write when fork() is called to create a new
# process. Pages that the parent and child processes share, such as the database
# connection, are marked read-only. If a write is performed on a read-only page
# (e.g., closing the connection), it is then copied, since the memory is no longer
# identical between the two processes. This leads to the following behavior:
#
# 1) Newly forked process
# parent
# -> connection (Django open, MySQL open)
# child
#
# 2) Child process closes the connection
# parent -> connection (*Django open, MySQL closed*)
# child -> connection (Django closed, MySQL closed)
#
# Calling connection.close() from a child process causes the MySQL server to
# close a connection which the parent process thinks is still usable. Since
# the parent process thinks the connection is still open, Django won't attempt
# to open a new one, and the parent ends up running a query on a closed connection.
# This results in a 'MySQL server has gone away' error.
#
# To resolve this, we force Django to reconnect to the database before running any queries.
connection.connect()
# If no courses exist for this partner, this command is likely being run on a
# new catalog installation. In that case, we don't want multiple threads racing
# to create courses. If courses do exist, this command is likely being run
# as an update, significantly lowering the probability of race conditions.
courses_exist = Course.objects.filter(partner=partner).exists()
is_threadsafe = courses_exist and waffle.switch_is_active(
'threaded_metadata_write')
max_workers = DataLoaderConfig.get_solo().max_workers
logger.info(
'Command is{negation} using threads to write data.'.format(
negation='' if is_threadsafe else ' not'))
pipeline = (
(
(SubjectMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers),
(SchoolMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers),
(SponsorMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers),
(PersonMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers),
),
(
(CourseMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers),
(OrganizationsApiDataLoader, partner.organizations_api_url,
max_workers),
),
((CoursesApiDataLoader, partner.courses_api_url,
max_workers), ),
(
(EcommerceApiDataLoader, partner.ecommerce_api_url, 1),
(ProgramsApiDataLoader, partner.programs_api_url,
max_workers),
),
((XSeriesMarketingSiteDataLoader,
partner.marketing_site_url_root, max_workers), ),
)
if waffle.switch_is_active('parallel_refresh_pipeline'):
for stage in pipeline:
with concurrent.futures.ProcessPoolExecutor() as executor:
for loader_class, api_url, max_workers in stage:
if api_url:
executor.submit(
execute_parallel_loader,
loader_class,
partner,
api_url,
access_token,
token_type,
max_workers,
is_threadsafe,
**kwargs,
)
else:
# Flatten pipeline and run serially.
for loader_class, api_url, max_workers in itertools.chain(
*(stage for stage in pipeline)):
if api_url:
execute_loader(
loader_class,
partner,
api_url,
access_token,
token_type,
max_workers,
is_threadsafe,
**kwargs,
)
def handle(self, *args, **options):
max_workers = options.get('max_workers')
# For each partner defined...
partners = Partner.objects.all()
# If a specific partner was indicated, filter down the set
partner_code = options.get('partner_code')
if partner_code:
partners = partners.filter(short_code=partner_code)
if not partners:
raise CommandError('No partners available!')
token_type = 'JWT'
for partner in partners:
logger.info('Retrieving access token for partner [{}]'.format(partner_code))
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=partner.oidc_url_root.strip('/')),
partner.oidc_key,
partner.oidc_secret,
token_type=token_type
)
except Exception:
logger.exception('No access token acquired through client_credential flow.')
raise
username = jwt.decode(access_token, verify=False)['preferred_username']
kwargs = {'username': username} if username else {}
# The Linux kernel implements copy-on-write when fork() is called to create a new
# process. Pages that the parent and child processes share, such as the database
# connection, are marked read-only. If a write is performed on a read-only page
# (e.g., closing the connection), it is then copied, since the memory is no longer
# identical between the two processes. This leads to the following behavior:
#
# 1) Newly forked process
# parent
# -> connection (Django open, MySQL open)
# child
#
# 2) Child process closes the connection
# parent -> connection (*Django open, MySQL closed*)
# child -> connection (Django closed, MySQL closed)
#
# Calling connection.close() from a child process causes the MySQL server to
# close a connection which the parent process thinks is still usable. Since
# the parent process thinks the connection is still open, Django won't attempt
# to open a new one, and the parent ends up running a query on a closed connection.
# This results in a 'MySQL server has gone away' error.
#
# To resolve this, we force Django to reconnect to the database before running any queries.
connection.connect()
# If no courses exist for this partner, this command is likely being run on a
# new catalog installation. In that case, we don't want multiple threads racing
# to create courses. If courses do exist, this command is likely being run
# as an update, significantly lowering the probability of race conditions.
courses_exist = Course.objects.filter(partner=partner).exists()
is_threadsafe = courses_exist and waffle.switch_is_active('threaded_metadata_write')
logger.info(
'Command is{negation} using threads to write data.'.format(negation='' if is_threadsafe else ' not')
)
pipeline = (
(
(SubjectMarketingSiteDataLoader, partner.marketing_site_url_root, None),
(SchoolMarketingSiteDataLoader, partner.marketing_site_url_root, None),
(SponsorMarketingSiteDataLoader, partner.marketing_site_url_root, None),
(PersonMarketingSiteDataLoader, partner.marketing_site_url_root, None),
),
(
(CourseMarketingSiteDataLoader, partner.marketing_site_url_root, None),
(OrganizationsApiDataLoader, partner.organizations_api_url, None),
),
(
(CoursesApiDataLoader, partner.courses_api_url, None),
),
(
(EcommerceApiDataLoader, partner.ecommerce_api_url, 1),
(ProgramsApiDataLoader, partner.programs_api_url, None),
),
(
(XSeriesMarketingSiteDataLoader, partner.marketing_site_url_root, None),
),
)
if waffle.switch_is_active('parallel_refresh_pipeline'):
for stage in pipeline:
with concurrent.futures.ProcessPoolExecutor() as executor:
for loader_class, api_url, max_workers_override in stage:
if api_url:
executor.submit(
execute_parallel_loader,
loader_class,
partner,
api_url,
access_token,
token_type,
(max_workers_override or max_workers),
is_threadsafe,
**kwargs,
)
else:
# Flatten pipeline and run serially.
for loader_class, api_url, max_workers_override in itertools.chain(*(stage for stage in pipeline)):
if api_url:
execute_loader(
loader_class,
partner,
api_url,
access_token,
token_type,
(max_workers_override or max_workers),
is_threadsafe,
**kwargs,
)
def test_get_client_credential_access_token_failure(self, code, body):
""" Test that the get access token method handles failure responses. """
with self.assertRaises(requests.RequestException):
self._mock_auth_api(URL, code, body=body)
EdxRestApiClient.get_oauth_access_token(URL, "client_id", "client_secret")
def process_load(args, sc, lms_url, temp_file):
""" Process load command
"""
# Try to get edX access token if not supplied
access_token = args.access_token
if not access_token:
logger.info(
'No access token provided. Retrieving access token using client_credential flow...'
)
try:
access_token, expires = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=args.oauth_host),
args.oauth_key,
args.oauth_secret,
token_type='jwt')
except Exception:
logger.exception(
'No access token provided or acquired through client_credential flow.'
)
raise
logger.info('Token retrieved: %s', access_token)
# use programs api to build table of course runs that are part of xseries
series_table = load_series_table()
# load any fixups
fixups = load_fixups(args.fixups)
logger.info(fixups)
client = EdxRestApiClient(args.content_api_url, jwt=access_token)
count = None
page = 1
course_runs = 0
# read the courses and create a Sailthru content item for each course_run within the course
while page:
# get a page of courses
response = client.courses().get(limit=500, offset=(page - 1) * 500)
count = response['count']
results = response['results']
if response['next']:
page += 1
else:
page = None
for course in results:
for course_run in course['course_runs']:
sailthru_content = create_sailthru_content(
course, course_run, series_table, lms_url, fixups)
if sailthru_content:
course_runs += 1
if sc:
try:
response = sc.api_post('content', sailthru_content)
except SailthruClientError as exc:
logger.exception(
"Exception attempting to update Sailthru", exc)
# wait 10 seconds and retry
time.sleep(10)
response = sc.api_post('content', sailthru_content)
if not response.is_ok():
logger.error(
"Error code %d connecting to Sailthru content api: %s",
response.json['error'],
response.json['errormsg'])
return
logger.info(
"Course: %s, Course_run: %s saved in Sailthru.",
course['key'], course_run['key'])
elif temp_file:
json.dump(sailthru_content, temp_file)
temp_file.write('\n')
logger.info(
"Course: %s, Course_run: %s being updated.",
course['key'], course_run['key'])
else:
logger.info(sailthru_content)
logger.info('Retrieved %d courses.', count)
logger.info('Saved %d course runs in Sailthru.', course_runs)
def test_get_client_credential_access_token_failure(self, code, body):
""" Test that the get access token method handles failure responses. """
with self.assertRaises(requests.RequestException):
self._mock_auth_api(OAUTH_URL, code, body=body)
EdxRestApiClient.get_oauth_access_token(OAUTH_URL, "client_id", "client_secret")
