def get_edx_api_data(api_config, resource, api, resource_id=None, querystring=None, cache_key=None, many=True, traverse_pagination=True): """GET data from an edX REST API. DRY utility for handling caching and pagination. Arguments: api_config (ConfigurationModel): The configuration model governing interaction with the API. resource (str): Name of the API resource being requested. Keyword Arguments: api (APIClient): API client to use for requesting data. resource_id (int or str): Identifies a specific resource to be retrieved. querystring (dict): Optional query string parameters. cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted (neither inspected nor updated). many (bool): Whether the resource requested is a collection of objects, or a single object. If false, an empty dict will be returned in cases of failure rather than the default empty list. traverse_pagination (bool): Whether to traverse pagination or return paginated response.. Returns: Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict) returned by DRF-powered APIs. """ no_data = [] if many else {} if not api_config.enabled: log.warning('%s configuration is disabled.', api_config.API_NAME) return no_data if cache_key: cache_key = '{}.{}'.format(cache_key, resource_id) if resource_id is not None else cache_key cache_key += '.zpickled' cached = cache.get(cache_key) if cached: return zunpickle(cached) try: endpoint = getattr(api, resource) querystring = querystring if querystring else {} response = endpoint(resource_id).get(**querystring) if resource_id is not None: results = response elif traverse_pagination: results = _traverse_pagination(response, endpoint, querystring, no_data) else: results = response except: # pylint: disable=bare-except log.exception('Failed to retrieve data from the %s API.', api_config.API_NAME) return no_data if cache_key: zdata = zpickle(results) cache.set(cache_key, zdata, api_config.cache_ttl) return results
def _deserialize(self, serialized_data, root_block_usage_key): """ Deserializes the given data and returns the parsed block_structure. """ block_relations, transformer_data, block_data_map = zunpickle(serialized_data) return BlockStructureFactory.create_new( root_block_usage_key, block_relations, transformer_data, block_data_map, )
def get(self, root_block_usage_key): """ Deserializes and returns the block structure starting at root_block_usage_key from the given cache, if it's found in the cache. The given root_block_usage_key must equate the root_block_usage_key previously passed to serialize_to_cache. Arguments: root_block_usage_key (UsageKey) - The usage_key for the root of the block structure that is to be deserialized from the given cache. Returns: BlockStructure - The deserialized block structure starting at root_block_usage_key, if found in the cache. NoneType - If the root_block_usage_key is not found in the cache. """ # Find root_block_usage_key in the cache. zp_data_from_cache = self._cache.get( self._encode_root_cache_key(root_block_usage_key)) if not zp_data_from_cache: logger.info( "Did not find BlockStructure %r in the cache.", root_block_usage_key, ) return None else: logger.info( "Read BlockStructure %r from cache, size: %s", root_block_usage_key, len(zp_data_from_cache), ) # Deserialize and construct the block structure. block_relations, transformer_data, block_data_map = zunpickle( zp_data_from_cache) block_structure = BlockStructureModulestoreData(root_block_usage_key) block_structure._block_relations = block_relations block_structure.transformer_data = transformer_data block_structure._block_data_map = block_data_map return block_structure
def get(self, root_block_usage_key): """ Deserializes and returns the block structure starting at root_block_usage_key from the given cache, if it's found in the cache. The given root_block_usage_key must equate the root_block_usage_key previously passed to serialize_to_cache. Arguments: root_block_usage_key (UsageKey) - The usage_key for the root of the block structure that is to be deserialized from the given cache. Returns: BlockStructure - The deserialized block structure starting at root_block_usage_key, if found in the cache. NoneType - If the root_block_usage_key is not found in the cache. """ # Find root_block_usage_key in the cache. zp_data_from_cache = self._cache.get(self._encode_root_cache_key(root_block_usage_key)) if not zp_data_from_cache: logger.info( "Did not find BlockStructure %r in the cache.", root_block_usage_key, ) return None else: logger.info( "Read BlockStructure %r from cache, size: %s", root_block_usage_key, len(zp_data_from_cache), ) # Deserialize and construct the block structure. block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache) block_structure = BlockStructureModulestoreData(root_block_usage_key) block_structure._block_relations = block_relations block_structure._transformer_data = transformer_data block_structure._block_data_map = block_data_map return block_structure
def _deserialize(self, serialized_data, root_block_usage_key): """ Deserializes the given data and returns the parsed block_structure. """ try: block_relations, transformer_data, block_data_map = zunpickle(serialized_data) except Exception: # Somehow failed to de-serialized the data, assume it's corrupt. bs_model = self._get_model(root_block_usage_key) logger.exception(u"BlockStructure: Failed to load data from cache for %s", bs_model) raise BlockStructureNotFound(bs_model.data_usage_key) return BlockStructureFactory.create_new( root_block_usage_key, block_relations, transformer_data, block_data_map, )
def get_api_data(api_config, resource, api_client, base_api_url, resource_id=None, querystring=None, cache_key=None, many=True, traverse_pagination=True, fields=None, long_term_cache=False): """ GET data from an edX REST API endpoint using the API client. DRY utility for handling caching and pagination. Arguments: api_config (ConfigurationModel): The configuration model governing interaction with the API. resource (str): Name of the API resource being requested. api_client (requests.Session): API client (either raw requests.Session or OAuthAPIClient) to use for requesting data. base_api_url (str): base API url, used to construct the full API URL across with resource and resource_id (if any). Keyword Arguments: resource_id (int or str): Identifies a specific resource to be retrieved. querystring (dict): Optional query string parameters. cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted (neither inspected nor updated). many (bool): Whether the resource requested is a collection of objects, or a single object. If false, an empty dict will be returned in cases of failure rather than the default empty list. traverse_pagination (bool): Whether to traverse pagination or return paginated response.. long_term_cache (bool): Whether to use the long term cache ttl or the standard cache ttl Returns: Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict) returned by DRF-powered APIs. """ no_data = [] if many else {} if not api_config.enabled: log.warning('%s configuration is disabled.', api_config.API_NAME) return no_data if cache_key: cache_key = f'{cache_key}.{resource_id}' if resource_id is not None else cache_key cache_key += '.zpickled' cached = cache.get(cache_key) if cached: try: cached_response = zunpickle(cached) except Exception: # pylint: disable=broad-except # Data is corrupt in some way. log.warning("Data for cache is corrupt for cache key %s", cache_key) cache.delete(cache_key) else: if fields: cached_response = get_fields(fields, cached_response) return cached_response try: querystring = querystring if querystring else {} api_url = urljoin( f"{base_api_url}/", f"{resource}/{str(resource_id) + '/' if resource_id is not None else ''}" ) response = api_client.get(api_url, params=querystring) response.raise_for_status() response = response.json() if resource_id is None and traverse_pagination: results = _traverse_pagination(response, api_client, api_url, querystring, no_data) else: results = response except: # pylint: disable=bare-except log.exception('Failed to retrieve data from the %s API.', api_config.API_NAME) return no_data if cache_key: zdata = zpickle(results) cache_ttl = api_config.cache_ttl if long_term_cache: cache_ttl = api_config.long_term_cache_ttl cache.set(cache_key, zdata, cache_ttl) if fields: results = get_fields(fields, results) return results
def get_edx_api_data(api_config, resource, api, resource_id=None, querystring=None, cache_key=None, many=True, traverse_pagination=True): """GET data from an edX REST API. DRY utility for handling caching and pagination. Arguments: api_config (ConfigurationModel): The configuration model governing interaction with the API. resource (str): Name of the API resource being requested. Keyword Arguments: api (APIClient): API client to use for requesting data. resource_id (int or str): Identifies a specific resource to be retrieved. querystring (dict): Optional query string parameters. cache_key (str): Where to cache retrieved data. The cache will be ignored if this is omitted (neither inspected nor updated). many (bool): Whether the resource requested is a collection of objects, or a single object. If false, an empty dict will be returned in cases of failure rather than the default empty list. traverse_pagination (bool): Whether to traverse pagination or return paginated response.. Returns: Data returned by the API. When hitting a list endpoint, extracts "results" (list of dict) returned by DRF-powered APIs. """ no_data = [] if many else {} if not api_config.enabled: log.warning('%s configuration is disabled.', api_config.API_NAME) return no_data if cache_key: cache_key = '{}.{}'.format( cache_key, resource_id) if resource_id is not None else cache_key cache_key += '.zpickled' cached = cache.get(cache_key) if cached: return zunpickle(cached) try: endpoint = getattr(api, resource) querystring = querystring if querystring else {} response = endpoint(resource_id).get(**querystring) if resource_id is not None: results = response elif traverse_pagination: results = _traverse_pagination(response, endpoint, querystring, no_data) else: results = response except: # pylint: disable=bare-except log.exception('Failed to retrieve data from the %s API.', api_config.API_NAME) return no_data if cache_key: zdata = zpickle(results) cache.set(cache_key, zdata, api_config.cache_ttl) return results
def create_from_cache(cls, root_block_usage_key, cache, transformers): """ Deserializes and returns the block structure starting at root_block_usage_key from the given cache, if it's found in the cache. The given root_block_usage_key must equate the root_block_usage_key previously passed to serialize_to_cache. Arguments: root_block_usage_key (UsageKey) - The usage_key for the root of the block structure that is to be deserialized from the given cache. cache (django.core.cache.backends.base.BaseCache) - The cache from which the block structure is to be deserialized. transformers ([BlockStructureTransformer]) - A list of transformers for which the block structure will be transformed. Returns: BlockStructure - The deserialized block structure starting at root_block_usage_key, if found in the cache. NoneType - If the root_block_usage_key is not found in the cache or if the cached data is outdated for one or more of the given transformers. """ # Find root_block_usage_key in the cache. zp_data_from_cache = cache.get(cls._encode_root_cache_key(root_block_usage_key)) if not zp_data_from_cache: logger.debug( "BlockStructure %r not found in the cache.", root_block_usage_key, ) return None else: logger.debug( "Read BlockStructure %r from cache, size: %s", root_block_usage_key, len(zp_data_from_cache), ) # Deserialize and construct the block structure. block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache) block_structure = BlockStructureBlockData(root_block_usage_key) block_structure._block_relations = block_relations block_structure._transformer_data = transformer_data block_structure._block_data_map = block_data_map # Verify that the cached data for all the given transformers are # for their latest versions. outdated_transformers = {} for transformer in transformers: cached_transformer_version = block_structure._get_transformer_data_version(transformer) if transformer.VERSION != cached_transformer_version: outdated_transformers[transformer.name()] = "version: {}, cached: {}".format( transformer.VERSION, cached_transformer_version, ) if outdated_transformers: logger.info( "Collected data for the following transformers are outdated:\n%s.", '\n'.join([t_name + ": " + t_value for t_name, t_value in outdated_transformers.iteritems()]), ) return None return block_structure
def create_from_cache(cls, root_block_usage_key, cache, transformers): """ Deserializes and returns the block structure starting at root_block_usage_key from the given cache, if it's found in the cache. The given root_block_usage_key must equate the root_block_usage_key previously passed to serialize_to_cache. Arguments: root_block_usage_key (UsageKey) - The usage_key for the root of the block structure that is to be deserialized from the given cache. cache (django.core.cache.backends.base.BaseCache) - The cache from which the block structure is to be deserialized. transformers ([BlockStructureTransformer]) - A list of transformers for which the block structure will be transformed. Returns: BlockStructure - The deserialized block structure starting at root_block_usage_key, if found in the cache. NoneType - If the root_block_usage_key is not found in the cache or if the cached data is outdated for one or more of the given transformers. """ # Find root_block_usage_key in the cache. zp_data_from_cache = cache.get( cls._encode_root_cache_key(root_block_usage_key)) if not zp_data_from_cache: logger.debug( "BlockStructure %r not found in the cache.", root_block_usage_key, ) return None else: logger.debug( "Read BlockStructure %r from cache, size: %s", root_block_usage_key, len(zp_data_from_cache), ) # Deserialize and construct the block structure. block_relations, transformer_data, block_data_map = zunpickle( zp_data_from_cache) block_structure = BlockStructureBlockData(root_block_usage_key) block_structure._block_relations = block_relations block_structure._transformer_data = transformer_data block_structure._block_data_map = block_data_map # Verify that the cached data for all the given transformers are # for their latest versions. outdated_transformers = {} for transformer in transformers: cached_transformer_version = block_structure._get_transformer_data_version( transformer) if transformer.VERSION != cached_transformer_version: outdated_transformers[ transformer.name()] = "version: {}, cached: {}".format( transformer.VERSION, cached_transformer_version, ) if outdated_transformers: logger.info( "Collected data for the following transformers are outdated:\n%s.", '\n'.join([ t_name + ": " + t_value for t_name, t_value in outdated_transformers.iteritems() ]), ) return None return block_structure