def test_with_required(self):
''' Test with required arguments supplied'''
initial = CatalogIntegration.current()
# test with both required args
call_command(
"create_catalog_integrations",
"--internal_api_url", self.catalog_integration_defaults['internal_api_url'],
"--service_username", self.catalog_integration_defaults['service_username']
)
current = CatalogIntegration.current()
# assert current has changed
self.assertNotEqual(
initial,
current
)
self.assertEqual(
current.enabled,
False
)
self.assertEqual(
current.internal_api_url,
self.catalog_integration_defaults['internal_api_url']
)
self.assertEqual(
current.service_username,
self.catalog_integration_defaults['service_username']
)
def test_cache_utilization(self):
"""Verify that when enabled, the cache is used."""
catalog_integration = self.create_catalog_integration(cache_ttl=5)
api = create_catalog_api_client(self.user)
expected_collection = ['some', 'test', 'data']
data = {
'next': None,
'results': expected_collection,
}
self._mock_catalog_api([
httpretty.Response(body=json.dumps(data),
content_type='application/json')
], )
resource_id = 1
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip(
'/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value'}
self._mock_catalog_api([
httpretty.Response(body=json.dumps(expected_resource),
content_type='application/json')
],
url=url)
cache_key = CatalogIntegration.current().CACHE_KEY
# Warm up the cache.
get_edx_api_data(catalog_integration,
'programs',
api=api,
cache_key=cache_key)
get_edx_api_data(catalog_integration,
'programs',
api=api,
resource_id=resource_id,
cache_key=cache_key)
# Hit the cache.
actual_collection = get_edx_api_data(catalog_integration,
'programs',
api=api,
cache_key=cache_key)
assert actual_collection == expected_collection
actual_resource = get_edx_api_data(catalog_integration,
'programs',
api=api,
resource_id=resource_id,
cache_key=cache_key)
assert actual_resource == expected_resource
# Verify that only two requests were made, not four.
self._assert_num_requests(2)
def create_catalog_integration(self, **kwargs):
"""
Creates a new CatalogIntegration with catalog_integration_defaults,
updated with any provided overrides.
"""
fields = dict(self.catalog_integration_defaults, **kwargs)
CatalogIntegration(**fields).save()
return CatalogIntegration.current()
def get_course_run(course_key, user):
"""Get a course run's data from the course catalog service.
Arguments:
course_key (CourseKey): Course key object identifying the run whose data we want.
user (User): The user to authenticate as when making requests to the catalog service.
Returns:
dict, empty if no data could be retrieved.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
api = create_catalog_api_client(user, catalog_integration)
data = get_edx_api_data(
catalog_integration,
user,
'course_runs',
resource_id=unicode(course_key),
cache_key=catalog_integration.CACHE_KEY
if catalog_integration.is_cache_enabled else None,
api=api,
querystring={'exclude_utm': 1},
)
return data if data else {}
else:
return {}
def get_program_types(name=None):
"""Retrieve program types from the catalog service.
Keyword Arguments:
name (string): Name identifying a specific program.
Returns:
list of dict, representing program types.
dict, if a specific program type is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
return []
api = create_catalog_api_client(user)
cache_key = '{base}.program_types'.format(
base=catalog_integration.CACHE_KEY)
data = get_edx_api_data(catalog_integration,
'program_types',
api=api,
cache_key=cache_key if
catalog_integration.is_cache_enabled else None)
# Filter by name if a name was provided
if name:
data = next(program_type for program_type in data
if program_type['name'] == name)
return data
else:
return []
def check_catalog_integration_and_get_user(error_message_field):
"""
Checks that catalog integration is enabled, and if so, attempts to get and
return the service user.
Parameters:
error_message_field (str): The field that will be attempted to be
retrieved when calling the api client. Used for the error message.
Returns:
Tuple of:
The catalog integration service user if it exists, else None
The catalog integration Object
Note: (This is necessary for future calls of functions using this method)
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.is_enabled():
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [{username}] does not exist. '
'{field} will not be retrieved.'.format(
username=catalog_integration.service_username,
field=error_message_field,
))
return None, catalog_integration
return user, catalog_integration
else:
logger.error(
'Unable to retrieve details about {field} because Catalog Integration is not enabled'
.format(field=error_message_field, ))
return None, catalog_integration
def get_course_run(course_key, user):
"""Get a course run's data from the course catalog service.
Arguments:
course_key (CourseKey): Course key object identifying the run whose data we want.
user (User): The user to authenticate as when making requests to the catalog service.
Returns:
dict, empty if no data could be retrieved.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
api = EdxRestApiClient(catalog_integration.internal_api_url, jwt=jwt)
data = get_edx_api_data(
catalog_integration,
user,
'course_runs',
resource_id=unicode(course_key),
cache_key=catalog_integration.CACHE_KEY
if catalog_integration.is_cache_enabled else None,
api=api,
)
return data if data else {}
else:
return {}
def get_course_run(course_key, user):
"""Get a course run's data from the course catalog service.
Arguments:
course_key (CourseKey): Course key object identifying the run whose data we want.
user (User): The user to authenticate as when making requests to the catalog service.
Returns:
dict, empty if no data could be retrieved.
"""
catalog_integration = CatalogIntegration.current()
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
api = EdxRestApiClient(catalog_integration.internal_api_url, jwt=jwt)
data = get_edx_api_data(
catalog_integration,
user,
'course_runs',
resource_id=unicode(course_key),
cache_key=catalog_integration.CACHE_KEY if catalog_integration.is_cache_enabled else None,
api=api,
)
return data if data else {}
def __init__(self):
"""
Initialize an authenticated Discovery service API client by using the
provided user.
"""
catalog_integration = CatalogIntegration.current()
# Client can't be used if there is no catalog integration
if not (catalog_integration and catalog_integration.enabled):
LOGGER.error(
"Unable to create DiscoveryApiClient because catalog integration not set up or enabled"
)
return None
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
LOGGER.error("Unable to retrieve catalog integration service user")
return None
jwt = JwtBuilder(user).build_token([])
base_url = configuration_helpers.get_value(
'COURSE_CATALOG_URL_BASE', settings.COURSE_CATALOG_URL_BASE)
self.client = EdxRestApiClient('{base_url}{journals_path}'.format(
base_url=base_url, journals_path=JOURNALS_API_PATH),
jwt=jwt)
def get_course_runs_for_course(course_uuid):
catalog_integration = CatalogIntegration.current()
if catalog_integration.is_enabled():
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
return []
api = create_catalog_api_client(user)
cache_key = '{base}.course.{uuid}.course_runs'.format(
base=catalog_integration.CACHE_KEY, uuid=course_uuid)
data = get_edx_api_data(
catalog_integration,
'courses',
resource_id=course_uuid,
api=api,
cache_key=cache_key
if catalog_integration.is_cache_enabled else None,
long_term_cache=True,
)
return data.get('course_runs', [])
else:
return []
def test_get_specific_resource(self):
"""Verify that a specific resource can be retrieved."""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
resource_id = 1
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip(
'/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value'}
self._mock_catalog_api([
httpretty.Response(body=json.dumps(expected_resource),
content_type='application/json')
],
url=url)
actual_resource = get_edx_api_data(catalog_integration,
'programs',
api=api,
resource_id=resource_id)
assert actual_resource == expected_resource
self._assert_num_requests(1)
def test_get_paginated_data(self):
"""Verify that paginated data can be retrieved."""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
expected_collection = ['some', 'test', 'data']
url = CatalogIntegration.current().get_internal_api_url().strip(
'/') + '/programs/?page={}'
responses = []
for page, record in enumerate(expected_collection, start=1):
data = {
'next':
url.format(page +
1) if page < len(expected_collection) else None,
'results': [record],
}
body = json.dumps(data)
responses.append(
httpretty.Response(body=body, content_type='application/json'))
self._mock_catalog_api(responses)
actual_collection = get_edx_api_data(catalog_integration,
'programs',
api=api)
assert actual_collection == expected_collection
self._assert_num_requests(len(expected_collection))
def get_course_run(course_key, user):
"""Get a course run's data from the course catalog service.
Arguments:
course_key (CourseKey): Course key object identifying the run whose data we want.
user (User): The user to authenticate as when making requests to the catalog service.
Returns:
dict, empty if no data could be retrieved.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
api = create_catalog_api_client(user, catalog_integration)
data = get_edx_api_data(
catalog_integration,
user,
'course_runs',
resource_id=unicode(course_key),
cache_key=catalog_integration.CACHE_KEY if catalog_integration.is_cache_enabled else None,
api=api,
)
return data if data else {}
else:
return {}
def get_program_types(name=None):
"""Retrieve program types from the catalog service.
Keyword Arguments:
name (string): Name identifying a specific program.
Returns:
list of dict, representing program types.
dict, if a specific program type is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
return []
api = create_catalog_api_client(user)
cache_key = '{base}.program_types'.format(base=catalog_integration.CACHE_KEY)
data = get_edx_api_data(catalog_integration, 'program_types', api=api,
cache_key=cache_key if catalog_integration.is_cache_enabled else None)
# Filter by name if a name was provided
if name:
data = next(program_type for program_type in data if program_type['name'] == name)
return data
else:
return []
def get_programs(uuid=None, types=None): # pylint: disable=redefined-builtin
"""Retrieve marketable programs from the catalog service.
Keyword Arguments:
uuid (string): UUID identifying a specific program.
types (list of string): List of program type names used to filter programs by type
(e.g., ["MicroMasters"] will only return MicroMasters programs,
["MicroMasters", "XSeries"] will return MicroMasters and XSeries programs).
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = User.objects.get(
username=catalog_integration.service_username)
except User.DoesNotExist:
return []
api = create_catalog_api_client(user, catalog_integration)
types_param = ','.join(types) if types else None
cache_key = '{base}.programs{types}'.format(
base=catalog_integration.CACHE_KEY,
types='.' + types_param if types_param else '')
querystring = {
'exclude_utm': 1,
}
# TODO ECOM-7650: Remove this after https://github.com/edx/course-discovery/pull/805 is merged and released.
if waffle.switch_is_active(
'display_retired_programs_on_learner_dashboard'):
querystring['status'] = (
'active',
'retired',
)
else:
querystring['marketable'] = 1
if uuid:
querystring['use_full_course_serializer'] = 1
if types_param:
querystring['types'] = types_param
return get_edx_api_data(
catalog_integration,
user,
'programs',
resource_id=uuid,
cache_key=cache_key
if catalog_integration.is_cache_enabled else None,
api=api,
querystring=querystring,
)
else:
return []
def test_get_specific_resource_with_falsey_id(self):
"""
Verify that a specific resource can be retrieved, and pagination parsing is
not attempted, if the ID passed is falsey (e.g., 0). The expected resource contains
a "results" key, as a paginatable item would have, so if the function looks for falsey
values in the resource_id field, rather than specifically None, the function will
return the value of that "results" key.
"""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
resource_id = 0
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip(
'/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value', 'results': []}
self._mock_catalog_api([
httpretty.Response(body=json.dumps(expected_resource),
content_type='application/json')
],
url=url)
actual_resource = get_edx_api_data(catalog_integration,
'programs',
api=api,
resource_id=resource_id)
assert actual_resource == expected_resource
self._assert_num_requests(1)
def get_course_run_details(course_run_key, fields):
"""
Retrieve information about the course run with the given id
Arguments:
course_run_key: key for the course_run about which we are retrieving information
Returns:
dict with language, start date, end date, and max_effort details about specified course run
"""
catalog_integration = CatalogIntegration.current()
course_run_details = dict()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
msg = 'Catalog service user {} does not exist. Data for course_run {} will not be retrieved'.format(
catalog_integration.service_username,
course_run_key
)
logger.error(msg)
return course_run_details
api = create_catalog_api_client(user)
cache_key = '{base}.course_runs'.format(base=catalog_integration.CACHE_KEY)
course_run_details = get_edx_api_data(catalog_integration, 'course_runs', api, resource_id=course_run_key,
cache_key=cache_key, many=False, traverse_pagination=False, fields=fields)
else:
msg = 'Unable to retrieve details about course_run {} because Catalog Integration is not enabled'.format(
course_run_key
)
logger.error(msg)
return course_run_details
def get_course_runs():
"""
Retrieve all the course runs from the catalog service.
Returns:
list of dict with each record representing a course run.
"""
catalog_integration = CatalogIntegration.current()
course_runs = []
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
return course_runs
api = create_catalog_api_client(user)
querystring = {
'page_size': catalog_integration.page_size,
'exclude_utm': 1,
}
course_runs = get_edx_api_data(catalog_integration, 'course_runs', api=api, querystring=querystring)
return course_runs
def get_course_runs():
"""
Retrieve all the course runs from the catalog service.
Returns:
list of dict with each record representing a course run.
"""
catalog_integration = CatalogIntegration.current()
course_runs = []
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
return course_runs
api = create_catalog_api_client(user)
querystring = {
'page_size': catalog_integration.page_size,
'exclude_utm': 1,
}
course_runs = get_edx_api_data(catalog_integration,
'course_runs',
api=api,
querystring=querystring)
return course_runs
def test_get_paginated_data_do_not_traverse_pagination(self):
"""
Verify that pagination is not traversed if traverse_pagination=False is passed as argument.
"""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
url = CatalogIntegration.current().get_internal_api_url().strip(
'/') + '/programs/?page={}'
responses = [
{
'next': url.format(2),
'results': ['some'],
},
{
'next': url.format(None),
'results': ['test'],
},
]
expected_response = responses[0]
self._mock_catalog_api([
httpretty.Response(body=json.dumps(body),
content_type='application/json')
for body in responses
])
actual_collection = get_edx_api_data(catalog_integration,
'programs',
api=api,
traverse_pagination=False)
assert actual_collection == expected_response
self._assert_num_requests(1)
def get_program_types():
"""Retrieve all program types from the catalog service.
Returns:
list of dict, representing program types.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = User.objects.get(username=catalog_integration.service_username)
except User.DoesNotExist:
return []
api = create_catalog_api_client(user, catalog_integration)
cache_key = '{base}.program_types'.format(base=catalog_integration.CACHE_KEY)
return get_edx_api_data(
catalog_integration,
user,
'program_types',
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
api=api
)
else:
return []
def get_course_runs_for_course(course_uuid):
catalog_integration = CatalogIntegration.current()
if catalog_integration.is_enabled():
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
return []
api = create_catalog_api_client(user)
cache_key = '{base}.course.{uuid}.course_runs'.format(
base=catalog_integration.CACHE_KEY,
uuid=course_uuid
)
data = get_edx_api_data(
catalog_integration,
'courses',
resource_id=course_uuid,
api=api,
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
long_term_cache=True
)
return data.get('course_runs', [])
else:
return []
def get_currency_data():
"""Retrieve currency data from the catalog service.
Returns:
list of dict, representing program types.
dict, if a specific program type is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
return []
api = create_catalog_api_client(user)
cache_key = '{base}.currency'.format(
base=catalog_integration.CACHE_KEY)
return get_edx_api_data(catalog_integration,
'currency',
api=api,
traverse_pagination=False,
cache_key=cache_key if
catalog_integration.is_cache_enabled else None)
else:
return []
def test_get_paginated_data_do_not_traverse_pagination(self):
"""
Verify that pagination is not traversed if traverse_pagination=False is passed as argument.
"""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
url = CatalogIntegration.current().get_internal_api_url().strip('/') + '/programs/?page={}'
responses = [
{
'next': url.format(2),
'results': ['some'],
},
{
'next': url.format(None),
'results': ['test'],
},
]
expected_response = responses[0]
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(body), content_type='application/json') for body in responses]
)
actual_collection = get_edx_api_data(catalog_integration, 'programs', api=api, traverse_pagination=False)
self.assertEqual(actual_collection, expected_response)
self._assert_num_requests(1)
def get_course_run_details(course_run_key, fields):
"""
Retrieve information about the course run with the given id
Arguments:
course_run_key: key for the course_run about which we are retrieving information
Returns:
dict with language, start date, end date, and max_effort details about specified course run
"""
catalog_integration = CatalogIntegration.current()
course_run_details = dict()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
msg = 'Catalog service user {} does not exist. Data for course_run {} will not be retrieved'.format(
catalog_integration.service_username,
course_run_key
)
logger.error(msg)
return course_run_details
api = create_catalog_api_client(user)
cache_key = '{base}.course_runs'.format(base=catalog_integration.CACHE_KEY)
course_run_details = get_edx_api_data(catalog_integration, 'course_runs', api, resource_id=course_run_key,
cache_key=cache_key, many=False, traverse_pagination=False, fields=fields)
else:
msg = 'Unable to retrieve details about course_run {} because Catalog Integration is not enabled'.format(
course_run_key
)
logger.error(msg)
return course_run_details
def get_program(program_uuid, ignore_cache=False):
"""
Retrieves the details for the specified program.
Args:
program_uuid (UUID): Program identifier
ignore_cache (bool): Indicates if previously-cached data should be ignored.
Returns:
dict
"""
program_uuid = str(program_uuid)
cache_key = 'programs.api.data.{uuid}'.format(uuid=program_uuid)
if not ignore_cache:
program = cache.get(cache_key)
if program:
return program
catalog_integration = CatalogIntegration.current()
user = catalog_integration.get_service_user()
api = create_catalog_api_client(user)
program = api.programs(program_uuid).get()
cache.set(cache_key, program, getattr(settings, 'PROGRAMS_CACHE_TTL', 60))
return program
def test_get_paginated_data(self):
"""Verify that paginated data can be retrieved."""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
expected_collection = ['some', 'test', 'data']
url = CatalogIntegration.current().get_internal_api_url().strip('/') + '/programs/?page={}'
responses = []
for page, record in enumerate(expected_collection, start=1):
data = {
'next': url.format(page + 1) if page < len(expected_collection) else None,
'results': [record],
}
body = json.dumps(data)
responses.append(
httpretty.Response(body=body, content_type='application/json')
)
self._mock_catalog_api(responses)
actual_collection = get_edx_api_data(catalog_integration, 'programs', api=api)
self.assertEqual(actual_collection, expected_collection)
self._assert_num_requests(len(expected_collection))
def test_get_specific_resource_with_falsey_id(self):
"""
Verify that a specific resource can be retrieved, and pagination parsing is
not attempted, if the ID passed is falsey (e.g., 0). The expected resource contains
a "results" key, as a paginatable item would have, so if the function looks for falsey
values in the resource_id field, rather than specifically None, the function will
return the value of that "results" key.
"""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
resource_id = 0
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip('/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value', 'results': []}
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(expected_resource), content_type='application/json')],
url=url
)
actual_resource = get_edx_api_data(catalog_integration, 'programs', api=api, resource_id=resource_id)
self.assertEqual(actual_resource, expected_resource)
self._assert_num_requests(1)
def _load_data(self, resource, default=DEFAULT_VALUE_SAFEGUARD, **kwargs):
"""
Load data from API client.
Arguments:
resource(string): type of resource to load
default(any): value to return if API query returned empty result. Sensible values: [], {}, None etc.
Returns:
dict: Deserialized response from Course Catalog API
"""
default_val = default if default != self.DEFAULT_VALUE_SAFEGUARD else {}
try:
return get_edx_api_data(
api_config=CatalogIntegration.current(),
resource=resource,
api=self.client,
**kwargs
) or default_val
except (SlumberBaseException, ConnectionError, Timeout) as exc:
LOGGER.exception(
'Failed to load data from resource [%s] with kwargs [%s] due to: [%s]',
resource, kwargs, str(exc)
)
return default_val
def test_with_optional(self):
''' Test with optionals arguments supplied'''
initial = CatalogIntegration.current()
# test --enabled
call_command(
"create_catalog_integrations",
"--internal_api_url", self.catalog_integration_defaults['internal_api_url'],
"--service_username", self.catalog_integration_defaults['service_username'],
"--enabled"
)
current = CatalogIntegration.current()
# assert current has changed
assert initial != current
assert current.enabled is True
assert current.internal_api_url == self.catalog_integration_defaults['internal_api_url']
assert current.service_username == self.catalog_integration_defaults['service_username']
# test with all args
call_command(
"create_catalog_integrations",
"--internal_api_url", self.catalog_integration_defaults['internal_api_url'],
"--service_username", self.catalog_integration_defaults['service_username'],
"--enabled",
"--cache_ttl", 500,
"--long_term_cache_ttl", 500,
"--page_size", 500
)
current = CatalogIntegration.current()
# assert current has changed
assert initial != current
assert current.enabled is True
assert current.internal_api_url == self.catalog_integration_defaults['internal_api_url']
assert current.service_username == self.catalog_integration_defaults['service_username']
assert current.cache_ttl == 500
assert current.long_term_cache_ttl == 500
assert current.page_size == 500
def test_with_optional(self):
''' Test with optionals arguments supplied'''
initial = CatalogIntegration.current()
# test --enabled
call_command("create_catalog_integrations", "--internal_api_url",
self.catalog_integration_defaults['internal_api_url'],
"--service_username",
self.catalog_integration_defaults['service_username'],
"--enabled")
current = CatalogIntegration.current()
# assert current has changed
self.assertNotEqual(initial, current)
self.assertEqual(current.enabled, True)
self.assertEqual(current.internal_api_url,
self.catalog_integration_defaults['internal_api_url'])
self.assertEqual(current.service_username,
self.catalog_integration_defaults['service_username'])
# test with all args
call_command("create_catalog_integrations", "--internal_api_url",
self.catalog_integration_defaults['internal_api_url'],
"--service_username",
self.catalog_integration_defaults['service_username'],
"--enabled", "--cache_ttl", 500, "--long_term_cache_ttl",
500, "--page_size", 500)
current = CatalogIntegration.current()
# assert current has changed
self.assertNotEqual(initial, current)
self.assertEqual(current.enabled, True)
self.assertEqual(current.internal_api_url,
self.catalog_integration_defaults['internal_api_url'])
self.assertEqual(current.service_username,
self.catalog_integration_defaults['service_username'])
self.assertEqual(current.cache_ttl, 500)
self.assertEqual(current.long_term_cache_ttl, 500)
self.assertEqual(current.page_size, 500)
def test_get_specific_fields_from_cache_response(self):
"""Verify that resource response is cached and get required fields from cached response"""
catalog_integration = self.create_catalog_integration(cache_ttl=5)
api = create_catalog_api_client(self.user)
response = {'lang': 'en', 'weeks_to_complete': '5'}
resource_id = 'course-v1:testX+testABC+1T2019'
url = '{api_root}/course_runs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip(
'/'),
resource_id=resource_id,
)
expected_resource_for_lang = {'lang': 'en'}
expected_resource_for_weeks_to_complete = {'weeks_to_complete': '5'}
self._mock_catalog_api([
httpretty.Response(body=json.dumps(response),
content_type='application/json')
],
url=url)
cache_key = CatalogIntegration.current().CACHE_KEY
# get response and set the cache.
actual_resource_for_lang = get_edx_api_data(catalog_integration,
'course_runs',
resource_id=resource_id,
api=api,
cache_key=cache_key,
fields=['lang'])
self.assertEqual(actual_resource_for_lang, expected_resource_for_lang)
# Hit the cache
actual_resource = get_edx_api_data(catalog_integration,
'course_runs',
api=api,
resource_id=resource_id,
cache_key=cache_key,
fields=['weeks_to_complete'])
self.assertEqual(actual_resource,
expected_resource_for_weeks_to_complete)
# Verify that only one requests were made, not three.
self._assert_num_requests(1)
def test_cache_utilization(self):
"""Verify that when enabled, the cache is used."""
catalog_integration = self.create_catalog_integration(cache_ttl=5)
api = create_catalog_api_client(self.user, catalog_integration)
expected_collection = ['some', 'test', 'data']
data = {
'next': None,
'results': expected_collection,
}
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(data), content_type='application/json')],
)
resource_id = 1
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().internal_api_url.strip('/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value'}
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(expected_resource), content_type='application/json')],
url=url
)
cache_key = CatalogIntegration.current().CACHE_KEY
# Warm up the cache.
get_edx_api_data(catalog_integration, self.user, 'programs', api=api, cache_key=cache_key)
get_edx_api_data(
catalog_integration, self.user, 'programs', api=api, resource_id=resource_id, cache_key=cache_key
)
# Hit the cache.
actual_collection = get_edx_api_data(catalog_integration, self.user, 'programs', api=api, cache_key=cache_key)
self.assertEqual(actual_collection, expected_collection)
actual_resource = get_edx_api_data(
catalog_integration, self.user, 'programs', api=api, resource_id=resource_id, cache_key=cache_key
)
self.assertEqual(actual_resource, expected_resource)
# Verify that only two requests were made, not four.
self._assert_num_requests(2)
def get_course_uuid_for_course(course_run_key):
"""
Retrieve the Course UUID for a given course key
Arguments:
course_run_key (CourseKey): A Key for a Course run that will be pulled apart to get just the information
required for a Course (e.g. org+course)
Returns:
UUID: Course UUID and None if it was not retrieved.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.is_enabled():
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [%s] does not exist. Course UUID will not be retrieved.',
catalog_integration.service_username,
)
return []
api = create_catalog_api_client(user)
run_cache_key = '{base}.course_run.{course_run_key}'.format(
base=catalog_integration.CACHE_KEY, course_run_key=course_run_key)
course_run_data = get_edx_api_data(
catalog_integration,
'course_runs',
resource_id=unicode(course_run_key),
api=api,
cache_key=run_cache_key
if catalog_integration.is_cache_enabled else None,
long_term_cache=True,
many=False,
)
course_key_str = course_run_data.get('course', None)
if course_key_str:
run_cache_key = '{base}.course.{course_key}'.format(
base=catalog_integration.CACHE_KEY, course_key=course_key_str)
data = get_edx_api_data(
catalog_integration,
'courses',
resource_id=course_key_str,
api=api,
cache_key=run_cache_key
if catalog_integration.is_cache_enabled else None,
long_term_cache=True,
many=False,
)
uuid_str = data.get('uuid', None)
if uuid_str:
return uuid.UUID(uuid_str)
return None
def handle(self, *args, **options):
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'
.format(username))
raise
try:
querystring = {
'exclude_utm': 1,
'status': ('active', 'retired'),
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
programs = {}
failure = False
for uuid in uuids:
try:
logger.info(
'Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception(
'Failed to retrieve details for program {uuid}.'.format(
uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(
successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def create_catalog_integration(self, **kwargs):
"""
Creates a new CatalogIntegration with DEFAULTS, updated with any provided overrides.
"""
fields = dict(self.DEFAULTS, **kwargs)
CatalogIntegration(**fields).save()
return CatalogIntegration.current()
def create_catalog_api_client(user):
"""Returns an API client which can be used to make Catalog API requests."""
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
url = CatalogIntegration.current().get_internal_api_url()
return EdxRestApiClient(url, jwt=jwt)
def get_catalog_api_base_url(site=None):
"""
Returns a base API url used to make Catalog API requests.
"""
if site:
return site.configuration.get_value('COURSE_CATALOG_API_URL')
return CatalogIntegration.current().get_internal_api_url()
def _mock_catalog_api(self, responses, url=None):
assert httpretty.is_enabled(
), 'httpretty must be enabled to mock Catalog API calls.'
url = url if url else CatalogIntegration.current(
).get_internal_api_url().strip('/') + '/programs/'
httpretty.register_uri(httpretty.GET, url, responses=responses)
def setUp(self):
super().setUp()
self.user = UserFactory()
self.base_api_url = CatalogIntegration.current().get_internal_api_url(
).strip('/')
httpretty.httpretty.reset()
cache.clear()
def cache_course_run(cls, catalog_course_run):
"""
Caches catalog course run for course key.
"""
cache.set(
cls._get_cache_key_name(catalog_course_run["key"]),
catalog_course_run,
CatalogIntegration.current().cache_ttl
)
def handle(self, *args, **options):
failure = False
logger.info('populate-multitenant-programs switch is ON')
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'
.format(username=username))
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value(
'COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(
domain=site.domain))
cache.set(
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(
domain=site.domain), [], None)
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(
client, site)
new_programs, program_details_failed = self.fetch_program_details(
client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info(
'Caching UUIDs for {total} programs for site {site_name}.'.
format(
total=len(uuids),
site_name=site.domain,
))
cache.set(
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain),
uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(
successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def create_catalog_api_client(user, site=None):
"""Returns an API client which can be used to make Catalog API requests."""
jwt = create_jwt_for_user(user)
if site:
url = site.configuration.get_value('COURSE_CATALOG_API_URL')
else:
url = CatalogIntegration.current().get_internal_api_url()
return EdxRestApiClient(url, jwt=jwt)
def get_programs(uuid=None, types=None): # pylint: disable=redefined-builtin
"""Retrieve marketable programs from the catalog service.
Keyword Arguments:
uuid (string): UUID identifying a specific program.
types (list of string): List of program type names used to filter programs by type
(e.g., ["MicroMasters"] will only return MicroMasters programs,
["MicroMasters", "XSeries"] will return MicroMasters and XSeries programs).
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = User.objects.get(username=catalog_integration.service_username)
except User.DoesNotExist:
return []
api = create_catalog_api_client(user, catalog_integration)
types_param = ','.join(types) if types else None
cache_key = '{base}.programs{types}'.format(
base=catalog_integration.CACHE_KEY,
types='.' + types_param if types_param else ''
)
querystring = {
'exclude_utm': 1,
}
# TODO ECOM-7650: Remove this after https://github.com/edx/course-discovery/pull/805 is merged and released.
if waffle.switch_is_active('display_retired_programs_on_learner_dashboard'):
querystring['status'] = ('active', 'retired',)
else:
querystring['marketable'] = 1
if uuid:
querystring['use_full_course_serializer'] = 1
if types_param:
querystring['types'] = types_param
return get_edx_api_data(
catalog_integration,
user,
'programs',
resource_id=uuid,
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
api=api,
querystring=querystring,
)
else:
return []
def create_catalog_api_client(user, site=None):
"""Returns an API client which can be used to make Catalog API requests."""
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
if site:
url = site.configuration.get_value('COURSE_CATALOG_API_URL')
else:
url = CatalogIntegration.current().get_internal_api_url()
return EdxRestApiClient(url, jwt=jwt)
def test_get_specific_fields_from_cache_response(self):
"""Verify that resource response is cached and get required fields from cached response"""
catalog_integration = self.create_catalog_integration(cache_ttl=5)
api = create_catalog_api_client(self.user)
response = {'lang': 'en', 'weeks_to_complete': '5'}
resource_id = 'course-v1:testX+testABC+1T2019'
url = '{api_root}/course_runs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip('/'),
resource_id=resource_id,
)
expected_resource_for_lang = {'lang': 'en'}
expected_resource_for_weeks_to_complete = {'weeks_to_complete': '5'}
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(response), content_type='application/json')],
url=url
)
cache_key = CatalogIntegration.current().CACHE_KEY
# get response and set the cache.
actual_resource_for_lang = get_edx_api_data(
catalog_integration, 'course_runs', resource_id=resource_id, api=api, cache_key=cache_key, fields=['lang']
)
self.assertEqual(actual_resource_for_lang, expected_resource_for_lang)
# Hit the cache
actual_resource = get_edx_api_data(
catalog_integration, 'course_runs', api=api, resource_id=resource_id, cache_key=cache_key,
fields=['weeks_to_complete']
)
self.assertEqual(actual_resource, expected_resource_for_weeks_to_complete)
# Verify that only one requests were made, not three.
self._assert_num_requests(1)
def get_programs(uuid=None, types=None): # pylint: disable=redefined-builtin
"""Retrieve marketable programs from the catalog service.
Keyword Arguments:
uuid (string): UUID identifying a specific program.
types (list of string): List of program type names used to filter programs by type
(e.g., ["MicroMasters"] will only return MicroMasters programs,
["MicroMasters", "XSeries"] will return MicroMasters and XSeries programs).
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = User.objects.get(username=catalog_integration.service_username)
except User.DoesNotExist:
return []
api = create_catalog_api_client(user, catalog_integration)
types_param = ','.join(types) if types else None
cache_key = '{base}.programs{types}'.format(
base=catalog_integration.CACHE_KEY,
types='.' + types_param if types_param else ''
)
querystring = {
'marketable': 1,
'exclude_utm': 1,
}
if uuid:
querystring['use_full_course_serializer'] = 1
if types_param:
querystring['types'] = types_param
return get_edx_api_data(
catalog_integration,
user,
'programs',
resource_id=uuid,
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
api=api,
querystring=querystring,
)
else:
return []
def bundle_about(request, bundle_uuid):
"""
Journal bundle about page's view.
"""
bundle = get_journal_bundles(request.site, bundle_uuid=bundle_uuid)
if not bundle:
raise Http404
bundle = bundle[0] # get_journal_bundles always returns list of bundles
bundle = extend_bundle(bundle)
context = {
'journals_root_url': get_journals_root_url(),
'discovery_root_url': CatalogIntegration.current().get_internal_api_url(),
'bundle': bundle,
'uses_bootstrap': True,
}
return render_to_response('journals/bundle_about.html', context)
def is_course_in_enterprise_catalog(site, course_id, enterprise_catalog_id):
"""
Verify that the provided course id exists in the site base list of course
run keys from the provided enterprise course catalog.
Arguments:
course_id (str): The course ID.
site: (django.contrib.sites.Site) site instance
enterprise_catalog_id (Int): Course catalog id of enterprise
Returns:
Boolean
"""
cache_key = get_cache_key(
site_domain=site.domain,
resource='catalogs.contains',
course_id=course_id,
catalog_id=enterprise_catalog_id
)
response = cache.get(cache_key)
if not response:
catalog_integration = CatalogIntegration.current()
if not catalog_integration.enabled:
LOGGER.error("Catalog integration is not enabled.")
return False
try:
user = User.objects.get(username=catalog_integration.service_username)
except User.DoesNotExist:
LOGGER.exception("Catalog service user '%s' does not exist.", catalog_integration.service_username)
return False
try:
# GET: /api/v1/catalogs/{catalog_id}/contains?course_run_id={course_run_ids}
response = create_catalog_api_client(user=user).catalogs(enterprise_catalog_id).contains.get(
course_run_id=course_id
)
cache.set(cache_key, response, settings.COURSES_API_CACHE_TIMEOUT)
except (ConnectionError, SlumberBaseException, Timeout):
LOGGER.exception('Unable to connect to Course Catalog service for catalog contains endpoint.')
return False
try:
return response['courses'][course_id]
except KeyError:
return False
def handle(self, *args, **options):
failure = False
logger.info('populate-multitenant-programs switch is ON')
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username=username)
)
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def get_programs(user=None, uuid=None, type=None): # pylint: disable=redefined-builtin
"""Retrieve marketable programs from the catalog service.
Keyword Arguments:
uuid (string): UUID identifying a specific program.
type (string): Filter programs by type (e.g., "MicroMasters" will only return MicroMasters programs).
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
user = _get_service_user(user, catalog_integration.service_username)
if not user:
return []
api = create_catalog_api_client(user, catalog_integration)
cache_key = '{base}.programs{type}'.format(
base=catalog_integration.CACHE_KEY,
type='.' + type if type else ''
)
querystring = {
'marketable': 1,
'exclude_utm': 1,
'published_course_runs_only': 1,
}
if type:
querystring['type'] = type
return get_edx_api_data(
catalog_integration,
user,
'programs',
resource_id=uuid,
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
api=api,
querystring=querystring,
)
else:
return []
def get_currency_data():
"""Retrieve currency data from the catalog service.
Returns:
list of dict, representing program types.
dict, if a specific program type is requested.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
return []
api = create_catalog_api_client(user)
cache_key = '{base}.currency'.format(base=catalog_integration.CACHE_KEY)
return get_edx_api_data(catalog_integration, 'currency', api=api,
cache_key=cache_key if catalog_integration.is_cache_enabled else None)
else:
return []
def get_and_cache_course_runs(course_keys, user):
"""
Get course run's data from the course catalog service and cache it.
"""
catalog_course_runs_against_course_keys = {}
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
api = create_catalog_api_client(user, catalog_integration)
catalog_data = get_edx_api_data(
catalog_integration,
user,
'course_runs',
api=api,
querystring={'keys': ",".join(course_keys), 'exclude_utm': 1},
)
if catalog_data:
for catalog_course_run in catalog_data:
CatalogCacheUtility.cache_course_run(catalog_course_run)
catalog_course_runs_against_course_keys[catalog_course_run["key"]] = catalog_course_run
return catalog_course_runs_against_course_keys
def test_get_specific_resource(self):
"""Verify that a specific resource can be retrieved."""
catalog_integration = self.create_catalog_integration()
api = create_catalog_api_client(self.user)
resource_id = 1
url = '{api_root}/programs/{resource_id}/'.format(
api_root=CatalogIntegration.current().get_internal_api_url().strip('/'),
resource_id=resource_id,
)
expected_resource = {'key': 'value'}
self._mock_catalog_api(
[httpretty.Response(body=json.dumps(expected_resource), content_type='application/json')],
url=url
)
actual_resource = get_edx_api_data(catalog_integration, 'programs', api=api, resource_id=resource_id)
self.assertEqual(actual_resource, expected_resource)
self._assert_num_requests(1)
def check_catalog_integration_and_get_user(error_message_field):
"""
Checks that catalog integration is enabled, and if so, attempts to get and
return the service user.
Parameters:
error_message_field (str): The field that will be attempted to be
retrieved when calling the api client. Used for the error message.
Returns:
Tuple of:
The catalog integration service user if it exists, else None
The catalog integration Object
Note: (This is necessary for future calls of functions using this method)
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.is_enabled():
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
logger.error(
'Catalog service user with username [{username}] does not exist. '
'{field} will not be retrieved.'.format(
username=catalog_integration.service_username,
field=error_message_field,
)
)
return None, catalog_integration
return user, catalog_integration
else:
logger.error(
'Unable to retrieve details about {field} because Catalog Integration is not enabled'.format(
field=error_message_field,
)
)
return None, catalog_integration
def get_program_types(user=None): # pylint: disable=redefined-builtin
"""Retrieve all program types from the catalog service.
Returns:
list of dict, representing program types.
"""
catalog_integration = CatalogIntegration.current()
if catalog_integration.enabled:
user = _get_service_user(user, catalog_integration.service_username)
if not user:
return []
api = create_catalog_api_client(user, catalog_integration)
cache_key = '{base}.program_types'.format(base=catalog_integration.CACHE_KEY)
return get_edx_api_data(
catalog_integration,
user,
'program_types',
cache_key=cache_key if catalog_integration.is_cache_enabled else None,
api=api
)
else:
return []
def __init__(self):
"""
Initialize an authenticated Discovery service API client by using the
provided user.
"""
catalog_integration = CatalogIntegration.current()
# Client can't be used if there is no catalog integration
if not (catalog_integration and catalog_integration.enabled):
LOGGER.error("Unable to create DiscoveryApiClient because catalog integration not set up or enabled")
return None
try:
user = catalog_integration.get_service_user()
except ObjectDoesNotExist:
LOGGER.error("Unable to retrieve catalog integration service user")
return None
jwt = JwtBuilder(user).build_token([])
base_url = configuration_helpers.get_value('COURSE_CATALOG_URL_BASE', settings.COURSE_CATALOG_URL_BASE)
self.client = EdxRestApiClient(
'{base_url}{journals_path}'.format(base_url=base_url, journals_path=JOURNALS_API_PATH),
jwt=jwt
)
def test_with_optional(self):
''' Test with optionals arguments supplied'''
initial = CatalogIntegration.current()
# test --enabled
call_command(
"create_catalog_integrations",
"--internal_api_url", self.catalog_integration_defaults['internal_api_url'],
"--service_username", self.catalog_integration_defaults['service_username'],
"--enabled"
)
current = CatalogIntegration.current()
# assert current has changed
self.assertNotEqual(
initial,
current
)
self.assertEqual(
current.enabled,
True
)
self.assertEqual(
current.internal_api_url,
self.catalog_integration_defaults['internal_api_url']
)
self.assertEqual(
current.service_username,
self.catalog_integration_defaults['service_username']
)
# test with all args
call_command(
"create_catalog_integrations",
"--internal_api_url", self.catalog_integration_defaults['internal_api_url'],
"--service_username", self.catalog_integration_defaults['service_username'],
"--enabled",
"--cache_ttl", 500,
"--long_term_cache_ttl", 500,
"--page_size", 500
)
current = CatalogIntegration.current()
# assert current has changed
self.assertNotEqual(
initial,
current
)
self.assertEqual(
current.enabled,
True
)
self.assertEqual(
current.internal_api_url,
self.catalog_integration_defaults['internal_api_url']
)
self.assertEqual(
current.service_username,
self.catalog_integration_defaults['service_username']
)
self.assertEqual(
current.cache_ttl,
500
)
self.assertEqual(
current.long_term_cache_ttl,
500
)
self.assertEqual(
current.page_size,
500
)
def handle(self, *args, **options):
if waffle.switch_is_active('populate-multitenant-programs'):
failure = False
logger.info('populate-multitenant-programs switch is ON')
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
else:
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
try:
querystring = {
'exclude_utm': 1,
'status': ('active', 'retired'),
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
programs = {}
failure = False
for uuid in uuids:
try:
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
