def add_caching_headers(response):
max_age = env.get_credential('FEC_CACHE_AGE')
cache_all_requests = env.get_credential('CACHE_ALL_REQUESTS', False)
status_code = response.status_code
if max_age is not None:
response.headers.add('Cache-Control', 'public, max-age={}'.format(max_age))
if (cache_all_requests and status_code == 200):
try:
# convert the results to JSON
json_data = utils.get_json_data(response)
# format the URL by removing the api_key and special characters
formatted_url = utils.format_url(request.url)
# get s3 bucket env variables
s3_bucket = utils.get_bucket()
cached_url = "s3://{0}/cached-calls/{1}.json".format(s3_bucket.name, formatted_url)
s3_key = utils.get_s3_key(cached_url)
# upload the request_content.json file to s3 bucket
with smart_open(s3_key, 'wb') as cached_file:
cached_file.write(json_data)
logger.info('The following request has been cached and uploaded successfully :%s ', cached_url)
except:
logger.error('Cache Upload failed')
return response
def add_caching_headers(response):
max_age = env.get_credential('FEC_CACHE_AGE')
cache_all_requests = env.get_credential('CACHE_ALL_REQUESTS', False)
status_code = response.status_code
if max_age is not None:
response.headers.add('Cache-Control',
'public, max-age={}'.format(max_age))
if (cache_all_requests and status_code == 200):
try:
# convert the results to JSON
json_data = utils.get_json_data(response)
# format the URL by removing the api_key and special characters
formatted_url = utils.format_url(request.url)
# get s3 bucket env variables
s3_bucket = utils.get_bucket()
cached_url = "s3://{0}/cached-calls/{1}.json".format(
s3_bucket.name, formatted_url)
s3_key = utils.get_s3_key(cached_url)
# upload the request_content.json file to s3 bucket
with smart_open(s3_key, 'wb') as cached_file:
cached_file.write(json_data)
logger.info(
'The following request has been cached and uploaded successfully :%s ',
cached_url)
except:
logger.error('Cache Upload failed')
return response
def test_ignore_case(self):
"""
format the URL when the API_KEY is all uppecase
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?API_KEY=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
expected_url = "schedules/schedule_e/by_candidate/candidate_id=s0al00156/cycle=2018/election_full=false/per_page=100"
self.assertEqual(url_after_format, expected_url)
def test_replace_special_chars_from_url(self):
"""
remove the special characters ? and & from the URL
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?api_key=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
expected_url = "schedules/schedule_e/by_candidate/candidate_id=s0al00156/cycle=2018/election_full=false/per_page=100"
self.assertEqual(url_after_format, expected_url)
def test_replace_special_chars_from_url(self):
"""
remove the special characters ?, & and = from the URL
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?api_key=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
expected_url = "schedules/schedule_e/by_candidate/candidate_id-S0AL00156-cycle-2018-election_full-false-per_page-100"
self.assertEqual(url_after_format, expected_url)
def test_ignore_case(self):
"""
format the URL when the API_KEY is all uppecase
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?API_KEY=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
expected_url = "schedules/schedule_e/by_candidate/candidate_id=s0al00156/cycle=2018/election_full=false/per_page=100"
self.assertEqual(url_after_format, expected_url)
def test_ignore_case(self):
"""
format the URL when the API_KEY is all uppecase
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?API_KEY=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
print('Formatted URL::', url_after_format)
expected_url = "schedules/schedule_e/by_candidate/API_KEY-DEMO_KEY-candidate_id-S0AL00156-cycle-2018-election_full-false-per_page-100"
self.assertEqual(url_after_format, expected_url)
def test_format_url(self):
"""
remove the api_key=DEMO_KEY
from the url with regex
"""
url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?api_key=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100"
url_after_format = utils.format_url(url_before_format)
expected_url = "schedules/schedule_e/by_candidate/candidate_id-S0AL00156-cycle-2018-election_full-false-per_page-100"
self.assertEqual(url_after_format, expected_url)
def add_caching_headers(response):
if app.config['MAX_CACHE_AGE'] is not None:
response.headers.add(
'Cache-Control',
'public, max-age={}'.format(app.config['MAX_CACHE_AGE']))
if (is_cacheable_endpoint(response.status_code, request.path)):
# Convert the response content into a JSON object and format the URL by
# removing the api_key parameter and other special characters.
json_data = utils.get_json_data(response)
formatted_url = utils.format_url(request.url)
logger.info('Attempting to cache request at: {}'.format(request.url))
cache_request.cache_all_requests.delay(json_data, formatted_url)
return response
def handle_exception(exception):
wrapped = ResponseException(str(exception), ErrorCode.INTERNAL_ERROR,
type(exception))
logger.info(
'An API error occurred with the status code of {status} ({exception}).'
.format(status=wrapped.status, exception=wrapped.wrappedException))
if is_retrievable_from_cache(wrapped.status, request.path):
logger.info('Attempting to retrieving the cached request from S3...')
# Retrieve the information needed to construct a URL for the S3 bucket
# where the cached API responses live.
formatted_url = utils.format_url(request.url)
s3_bucket = utils.get_bucket()
bucket_region = env.get_credential('region')
cached_url = "http://s3-{0}.amazonaws.com/{1}/cached-calls/{2}".format(
bucket_region, s3_bucket.name, formatted_url)
# Attempt to retrieve the cached data from S3.
cached_data = utils.get_cached_request(cached_url)
# If the cached data was returned, we can return that to the client.
# Otherwise, log the error and raise an API error.
if cached_data is not None:
logger.info('Successfully retrieved cached request from S3.')
return cached_data
else:
logger.error(
'An error occured while retrieving the cached file from S3.')
raise exceptions.ApiError(
'The requested URL could not be found.'.format(request.url),
status_code=http.client.NOT_FOUND)
else:
raise exceptions.ApiError(
'The requested URL could not be found.'.format(request.url),
status_code=http.client.NOT_FOUND)
