def get_edx_api_data(api_config, resource, api, resource_id=None, querystring=None, cache_key=None, many=True,
traverse_pagination=True):
"""GET data from an edX REST API.
DRY utility for handling caching and pagination.
Arguments:
api_config (ConfigurationModel): The configuration model governing interaction with the API.
resource (str): Name of the API resource being requested.
Keyword Arguments:
api (APIClient): API client to use for requesting data.
resource_id (int or str): Identifies a specific resource to be retrieved.
querystring (dict): Optional query string parameters.
cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted
(neither inspected nor updated).
many (bool): Whether the resource requested is a collection of objects, or a single object.
If false, an empty dict will be returned in cases of failure rather than the default empty list.
traverse_pagination (bool): Whether to traverse pagination or return paginated response..
Returns:
Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict)
returned by DRF-powered APIs.
"""
no_data = [] if many else {}
if not api_config.enabled:
log.warning('%s configuration is disabled.', api_config.API_NAME)
return no_data
if cache_key:
cache_key = '{}.{}'.format(cache_key, resource_id) if resource_id is not None else cache_key
cache_key += '.zpickled'
cached = cache.get(cache_key)
if cached:
return zunpickle(cached)
try:
endpoint = getattr(api, resource)
querystring = querystring if querystring else {}
response = endpoint(resource_id).get(**querystring)
if resource_id is not None:
results = response
elif traverse_pagination:
results = _traverse_pagination(response, endpoint, querystring, no_data)
else:
results = response
except: # pylint: disable=bare-except
log.exception('Failed to retrieve data from the %s API.', api_config.API_NAME)
return no_data
if cache_key:
zdata = zpickle(results)
cache.set(cache_key, zdata, api_config.cache_ttl)
return results
def _deserialize(self, serialized_data, root_block_usage_key):
"""
Deserializes the given data and returns the parsed block_structure.
"""
block_relations, transformer_data, block_data_map = zunpickle(serialized_data)
return BlockStructureFactory.create_new(
root_block_usage_key,
block_relations,
transformer_data,
block_data_map,
)
def get(self, root_block_usage_key):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
The given root_block_usage_key must equate the root_block_usage_key
previously passed to serialize_to_cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be deserialized from
the given cache.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache.
"""
# Find root_block_usage_key in the cache.
zp_data_from_cache = self._cache.get(
self._encode_root_cache_key(root_block_usage_key))
if not zp_data_from_cache:
logger.info(
"Did not find BlockStructure %r in the cache.",
root_block_usage_key,
)
return None
else:
logger.info(
"Read BlockStructure %r from cache, size: %s",
root_block_usage_key,
len(zp_data_from_cache),
)
# Deserialize and construct the block structure.
block_relations, transformer_data, block_data_map = zunpickle(
zp_data_from_cache)
block_structure = BlockStructureModulestoreData(root_block_usage_key)
block_structure._block_relations = block_relations
block_structure.transformer_data = transformer_data
block_structure._block_data_map = block_data_map
return block_structure
def get(self, root_block_usage_key):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
The given root_block_usage_key must equate the root_block_usage_key
previously passed to serialize_to_cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be deserialized from
the given cache.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache.
"""
# Find root_block_usage_key in the cache.
zp_data_from_cache = self._cache.get(self._encode_root_cache_key(root_block_usage_key))
if not zp_data_from_cache:
logger.info(
"Did not find BlockStructure %r in the cache.",
root_block_usage_key,
)
return None
else:
logger.info(
"Read BlockStructure %r from cache, size: %s",
root_block_usage_key,
len(zp_data_from_cache),
)
# Deserialize and construct the block structure.
block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
block_structure = BlockStructureModulestoreData(root_block_usage_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
block_structure._block_data_map = block_data_map
return block_structure
def _deserialize(self, serialized_data, root_block_usage_key):
"""
Deserializes the given data and returns the parsed block_structure.
"""
try:
block_relations, transformer_data, block_data_map = zunpickle(serialized_data)
except Exception:
# Somehow failed to de-serialized the data, assume it's corrupt.
bs_model = self._get_model(root_block_usage_key)
logger.exception(u"BlockStructure: Failed to load data from cache for %s", bs_model)
raise BlockStructureNotFound(bs_model.data_usage_key)
return BlockStructureFactory.create_new(
root_block_usage_key,
block_relations,
transformer_data,
block_data_map,
)
def get_api_data(api_config,
resource,
api_client,
base_api_url,
resource_id=None,
querystring=None,
cache_key=None,
many=True,
traverse_pagination=True,
fields=None,
long_term_cache=False):
"""
GET data from an edX REST API endpoint using the API client.
DRY utility for handling caching and pagination.
Arguments:
api_config (ConfigurationModel): The configuration model governing interaction with the API.
resource (str): Name of the API resource being requested.
api_client (requests.Session): API client (either raw requests.Session or OAuthAPIClient) to use for
requesting data.
base_api_url (str): base API url, used to construct the full API URL across with resource and
resource_id (if any).
Keyword Arguments:
resource_id (int or str): Identifies a specific resource to be retrieved.
querystring (dict): Optional query string parameters.
cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted
(neither inspected nor updated).
many (bool): Whether the resource requested is a collection of objects, or a single object.
If false, an empty dict will be returned in cases of failure rather than the default empty list.
traverse_pagination (bool): Whether to traverse pagination or return paginated response..
long_term_cache (bool): Whether to use the long term cache ttl or the standard cache ttl
Returns:
Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict)
returned by DRF-powered APIs.
"""
no_data = [] if many else {}
if not api_config.enabled:
log.warning('%s configuration is disabled.', api_config.API_NAME)
return no_data
if cache_key:
cache_key = f'{cache_key}.{resource_id}' if resource_id is not None else cache_key
cache_key += '.zpickled'
cached = cache.get(cache_key)
if cached:
try:
cached_response = zunpickle(cached)
except Exception: # pylint: disable=broad-except
# Data is corrupt in some way.
log.warning("Data for cache is corrupt for cache key %s",
cache_key)
cache.delete(cache_key)
else:
if fields:
cached_response = get_fields(fields, cached_response)
return cached_response
try:
querystring = querystring if querystring else {}
api_url = urljoin(
f"{base_api_url}/",
f"{resource}/{str(resource_id) + '/' if resource_id is not None else ''}"
)
response = api_client.get(api_url, params=querystring)
response.raise_for_status()
response = response.json()
if resource_id is None and traverse_pagination:
results = _traverse_pagination(response, api_client, api_url,
querystring, no_data)
else:
results = response
except: # pylint: disable=bare-except
log.exception('Failed to retrieve data from the %s API.',
api_config.API_NAME)
return no_data
if cache_key:
zdata = zpickle(results)
cache_ttl = api_config.cache_ttl
if long_term_cache:
cache_ttl = api_config.long_term_cache_ttl
cache.set(cache_key, zdata, cache_ttl)
if fields:
results = get_fields(fields, results)
return results
def get_edx_api_data(api_config,
resource,
api,
resource_id=None,
querystring=None,
cache_key=None,
many=True,
traverse_pagination=True):
"""GET data from an edX REST API.
DRY utility for handling caching and pagination.
Arguments:
api_config (ConfigurationModel): The configuration model governing interaction with the API.
resource (str): Name of the API resource being requested.
Keyword Arguments:
api (APIClient): API client to use for requesting data.
resource_id (int or str): Identifies a specific resource to be retrieved.
querystring (dict): Optional query string parameters.
cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted
(neither inspected nor updated).
many (bool): Whether the resource requested is a collection of objects, or a single object.
If false, an empty dict will be returned in cases of failure rather than the default empty list.
traverse_pagination (bool): Whether to traverse pagination or return paginated response..
Returns:
Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict)
returned by DRF-powered APIs.
"""
no_data = [] if many else {}
if not api_config.enabled:
log.warning('%s configuration is disabled.', api_config.API_NAME)
return no_data
if cache_key:
cache_key = '{}.{}'.format(
cache_key, resource_id) if resource_id is not None else cache_key
cache_key += '.zpickled'
cached = cache.get(cache_key)
if cached:
return zunpickle(cached)
try:
endpoint = getattr(api, resource)
querystring = querystring if querystring else {}
response = endpoint(resource_id).get(**querystring)
if resource_id is not None:
results = response
elif traverse_pagination:
results = _traverse_pagination(response, endpoint, querystring,
no_data)
else:
results = response
except: # pylint: disable=bare-except
log.exception('Failed to retrieve data from the %s API.',
api_config.API_NAME)
return no_data
if cache_key:
zdata = zpickle(results)
cache.set(cache_key, zdata, api_config.cache_ttl)
return results
def create_from_cache(cls, root_block_usage_key, cache, transformers):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
The given root_block_usage_key must equate the root_block_usage_key
previously passed to serialize_to_cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be deserialized from
the given cache.
cache (django.core.cache.backends.base.BaseCache) - The
cache from which the block structure is to be
deserialized.
transformers ([BlockStructureTransformer]) - A list of
transformers for which the block structure will be
transformed.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache
or if the cached data is outdated for one or more of the
given transformers.
"""
# Find root_block_usage_key in the cache.
zp_data_from_cache = cache.get(cls._encode_root_cache_key(root_block_usage_key))
if not zp_data_from_cache:
logger.debug(
"BlockStructure %r not found in the cache.",
root_block_usage_key,
)
return None
else:
logger.debug(
"Read BlockStructure %r from cache, size: %s",
root_block_usage_key,
len(zp_data_from_cache),
)
# Deserialize and construct the block structure.
block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
block_structure = BlockStructureBlockData(root_block_usage_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
block_structure._block_data_map = block_data_map
# Verify that the cached data for all the given transformers are
# for their latest versions.
outdated_transformers = {}
for transformer in transformers:
cached_transformer_version = block_structure._get_transformer_data_version(transformer)
if transformer.VERSION != cached_transformer_version:
outdated_transformers[transformer.name()] = "version: {}, cached: {}".format(
transformer.VERSION,
cached_transformer_version,
)
if outdated_transformers:
logger.info(
"Collected data for the following transformers are outdated:\n%s.",
'\n'.join([t_name + ": " + t_value for t_name, t_value in outdated_transformers.iteritems()]),
)
return None
return block_structure
def create_from_cache(cls, root_block_usage_key, cache, transformers):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
The given root_block_usage_key must equate the root_block_usage_key
previously passed to serialize_to_cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be deserialized from
the given cache.
cache (django.core.cache.backends.base.BaseCache) - The
cache from which the block structure is to be
deserialized.
transformers ([BlockStructureTransformer]) - A list of
transformers for which the block structure will be
transformed.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache
or if the cached data is outdated for one or more of the
given transformers.
"""
# Find root_block_usage_key in the cache.
zp_data_from_cache = cache.get(
cls._encode_root_cache_key(root_block_usage_key))
if not zp_data_from_cache:
logger.debug(
"BlockStructure %r not found in the cache.",
root_block_usage_key,
)
return None
else:
logger.debug(
"Read BlockStructure %r from cache, size: %s",
root_block_usage_key,
len(zp_data_from_cache),
)
# Deserialize and construct the block structure.
block_relations, transformer_data, block_data_map = zunpickle(
zp_data_from_cache)
block_structure = BlockStructureBlockData(root_block_usage_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
block_structure._block_data_map = block_data_map
# Verify that the cached data for all the given transformers are
# for their latest versions.
outdated_transformers = {}
for transformer in transformers:
cached_transformer_version = block_structure._get_transformer_data_version(
transformer)
if transformer.VERSION != cached_transformer_version:
outdated_transformers[
transformer.name()] = "version: {}, cached: {}".format(
transformer.VERSION,
cached_transformer_version,
)
if outdated_transformers:
logger.info(
"Collected data for the following transformers are outdated:\n%s.",
'\n'.join([
t_name + ": " + t_value
for t_name, t_value in outdated_transformers.iteritems()
]),
)
return None
return block_structure
