def test_spectre_404s_when_destination_is_missing(self):
response = get_through_spectre('/not_cacheable')
assert response.status_code == 200
response = get_through_spectre('/not_cacheable', extra_headers={'X-Smartstack-Destination': None})
assert response.status_code == 404
assert 'Not found: GET /not_cacheable' in response.text
def test_default_vary_headers(self):
assert get_through_spectre('/long_ttl/vary').headers['Spectre-Cache-Status'] == 'miss'
assert assert_is_in_spectre_cache('/long_ttl/vary')
# namespace vary_headers do not contain X-Mode
assert assert_is_in_spectre_cache('/long_ttl/vary', {'x-mode': 'ro'})
assert get_through_spectre('/long_ttl/vary', {'accept-encoding': 'text'}).headers['Spectre-Cache-Status'] == 'miss'
def test_purge_cassandra_by_id(self):
get_through_spectre('/bulk_requester?ids=1')
assert_is_in_spectre_cache('/bulk_requester?ids=1')
# Purge id 1
purge_resource({'namespace': 'backend.main', 'cache_name': 'bulk_requester_does_not_cache_missing_ids', 'id': '1'})
get_resp_3 = get_through_spectre('/bulk_requester?ids=1')
# resp 3 was no longer cached.
assert get_resp_3.headers['Spectre-Cache-Status'] == 'miss'
def test_response_headers_passed_back(self):
response = get_through_spectre('/not_cacheable')
assert response.headers['Some-Header'] == 'abc'
assert response.headers['Spectre-Cache-Status'] == 'non-cacheable-uri (backend.main)'
response = get_through_spectre('/timestamp/response_header')
assert response.headers['Spectre-Cache-Status'] == 'miss'
assert response.headers['Some-Header'] == 'abc'
response = assert_is_in_spectre_cache('/timestamp/response_header')
assert response.headers['Some-Header'] == 'abc'
def test_different_params_same_id(self):
base_path = '/bulk_requester'
resp1 = get_through_spectre("{}?ids={}&data=false".format(base_path, '5%2C6'))
assert resp1.headers['Spectre-Cache-Status'] == 'miss'
resp2 = get_through_spectre("{}?ids={}&data=true".format(base_path, '5'))
assert resp2.headers['Spectre-Cache-Status'] == 'miss'
assert_is_in_spectre_cache("{}?data=false&ids={}".format(base_path, '5'))
assert_is_in_spectre_cache("{}?data=false&ids={}".format(base_path, '6'))
assert_is_in_spectre_cache("{}?ids={}&data=false".format(base_path, '6'))
def test_non_application_json(self):
extra_header = {'test-content-type': 'text'}
base_path = '/bulk_requester'
resp = get_through_spectre("{}?ids={}".format(base_path, '4%2C5'), extra_headers=extra_header)
assert resp.text == 'this is text'
assert resp.status_code == 200
assert resp.headers['Spectre-Cache-Status'] == "unable to process response; content-type is text"
# Single id isn't cached either
resp = get_through_spectre("{}?ids={}".format(base_path, '4'), extra_headers=extra_header)
assert resp.text == 'this is text'
assert resp.status_code == 200
assert resp.headers['Spectre-Cache-Status'] == "unable to process response; content-type is text"
def test_spectre_status_response_header(self):
response = get_through_spectre('/not_cacheable')
assert response.headers['Spectre-Cache-Status'] == 'non-cacheable-uri (backend.main)'
response = get_through_spectre('/timestamp/spectre_status_header')
assert response.headers['Spectre-Cache-Status'] == 'miss'
assert_is_in_spectre_cache('/timestamp/spectre_status_header')
response = get_through_spectre('/timestamp/spectre_status_header', extra_headers={'Pragma': 'no-cache'})
assert response.headers['Spectre-Cache-Status'] == 'no-cache-header'
response = post_through_spectre('/timestamp/spectre_status_header')
assert response.headers['Spectre-Cache-Status'] == 'non-cacheable-method'
def test_caching_json_with_null(self):
response1 = get_through_spectre('/timestamp/cached')
time.sleep(0.2)
response2 = get_through_spectre('/timestamp/cached')
body1 = response1.json()
body2 = response2.json()
# keys with null values should be persisted
assert 'null_value' in body1
assert 'null_value' in body2
assert body1['null_value'] is None
assert body2['null_value'] is None
def test_caching_json_with_null(self):
base_path = '/bulk_requester_2'
resp_1 = get_through_spectre("{}/{}/v1?k1=v2".format(base_path, '1,2,3'))
resp_2 = get_through_spectre("{}/{}/v1?k1=v2".format(base_path, '2'))
_, bulk_body = resp_1.headers, resp_1.json()
_, body = resp_2.headers, resp_2.json()
# keys with null values should be persisted
assert 'null_value' in bulk_body[0]
assert 'null_value' in bulk_body[1]
assert 'null_value' in bulk_body[2]
assert 'null_value' in body[0]
assert bulk_body[1]['null_value'] is None
assert body[0]['null_value'] is None
def test_uncacheable_headers_not_passed_back(self):
"""Spectre doesn't store uncacheable headers in the cache, but it always
passes back all response headers in the uncached (or uncacheable) case.
"""
response = get_through_spectre('/not_cacheable')
assert response.headers['Spectre-Cache-Status'] == 'non-cacheable-uri (backend.main)'
assert 'Uncacheable-Header' in response.headers
response = get_through_spectre('/timestamp/uncacheable_header')
assert response.headers['Spectre-Cache-Status'] == 'miss'
assert 'Uncacheable-Header' in response.headers
# This is the cached case, where the uncacheable header isn't expected
# in the response.
response = assert_is_in_spectre_cache('/timestamp/uncacheable_header')
assert 'Uncacheable-Header' not in response.headers
def test_different_vary_headers(self):
# same vary header
val1 = get_through_spectre('/timestamp/cached', {'accept-encoding': 'testzip, deflate, custom'})
assert val1.headers['Spectre-Cache-Status'] == 'miss'
assert_is_in_spectre_cache('/timestamp/cached', {'accept-encoding': 'testzip, deflate, custom'})
# different vary headers --> val3 is a miss
val3 = get_through_spectre('/timestamp/cached', {'accept-encoding': 'none'})
assert val3.headers['Spectre-Cache-Status'] == 'miss'
# contains x-mode --> miss
val4 = get_through_spectre('/timestamp/cached', {'X-Mode': 'ro'})
assert val4.headers['Spectre-Cache-Status'] == 'miss'
# Host is not a Vary header --> hit
assert_is_in_spectre_cache('/timestamp/cached', {'X-Mode': 'ro', 'Host': 'localhost'})
def test_no_cache_header_metrics(log_file):
response = get_through_spectre(
'/timestamp/',
extra_headers={'Pragma': 'spectre-no-cache'},
)
assert response.status_code == 200
metrics = _load_metrics(log_file)
# Since we send the no-cache header we don't have a `spectre.fetch_body_and_headers`
# or `spectre.hit_rate`. We still update the cache though, so we have the
# `spectre.store_body_and_headers`
_assert_store_metric(metrics[0], 'timestamp')
# Finally we emit the `spectre.no_cache_header`
assert metrics[1].dimensions == {
'metric_name': 'spectre.no_cache_header',
'habitat': 'uswest1a',
'service_name': 'spectre',
'namespace': 'backend.main',
'instance_name': 'itest',
'reason': 'no-cache-header',
'cache_name': 'timestamp',
}
# This assert is mostly there to make sure there are no more metrics than what I expect.
# The reason why it's not before the other asserts is because the error message doesn't
# show you what metrics are actually in the list, so it's very annoying to figure out
# what's missing. You'd need to comment out this check and then verify which of the
# other asserts is failing.
assert len(metrics) == 2
def test_cache_miss(log_file):
response = get_through_spectre(
'/timestamp/',
)
assert response.status_code == 200
assert response.headers['Spectre-Cache-Status'] == 'miss'
metrics = _load_metrics(log_file)
# First 2 metrics are `spectre.fetch_body_and_headers` and `spectre.hit_rate`
_assert_fetch_hit_rate(metrics[0:2], 'timestamp')
# Then since it's a miss we have a `spectre.store_body_and_headers`
_assert_store_metric(metrics[2], 'timestamp')
# Finally the `spectre.request_timing`
_assert_request_timing_metrics(metrics[3:7], 'timestamp')
# This assert is mostly there to make sure there are no more metrics than what I expect.
# The reason why it's not before the other asserts is because the error message doesn't
# show you what metrics are actually in the list, so it's very annoying to figure out
# what's missing. You'd need to comment out this check and then verify which of the
# other asserts is failing.
assert len(metrics) == 7
def test_dont_emit_span_if_no_headers(self, clean_log_files):
response = get_through_spectre(
'/not_cacheable',
)
assert response.status_code == 200
self._assert_no_span_in_logs()
def test_hop_by_hop_headers(self):
response = get_through_spectre('/timestamp/get')
assert response.status_code == 200
# Our test backend sends 'Connection: close'. Connection being a
# hop-by-hop header, spectre should not forward it and we expect to see
# openresty's "keep-alive" instead.
# See http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1
assert response.headers['Connection'] == 'keep-alive'
def test_caching_works_with_id_extraction(self):
response = get_through_spectre('/biz?foo=bar&business_id=1234')
assert response.headers['Spectre-Cache-Status'] == 'miss'
# ensure extracting the id is not messing up the caching logic
assert_is_in_spectre_cache('/biz?foo=bar&business_id=1234')
# check that invalidation is actually supported
purge_resource({
'namespace': 'backend.main',
'cache_name': 'url_with_id_extraction',
'id': '1234',
})
# now this should be a cache miss
response = get_through_spectre('/biz?foo=bar&business_id=1234')
assert response.headers['Spectre-Cache-Status'] == 'miss'
def test_http_timeouts_return_504(self):
start = time.time()
response = get_through_spectre('/timestamp/cached?sleep=1500')
duration = time.time() - start
assert response.text == 'Error requesting /timestamp/cached?sleep=1500: timeout'
assert response.status_code == 504
# Duration should be >= 1.0 seconds since that's the HTTP_TIMEOUT_MS
# value we set in start.sh
assert duration >= 1.0
def test_with_invalid_id_when_cache_missing_ids_is_true(self):
path = '/bulk_requester_2/{ids}/v1?k1=v1'
# 0 < ids < 1000 are valid
response = get_through_spectre(path.format(ids='10,5000,11'))
assert len(response.json()) == 2
assert response.headers['Spectre-Cache-Status'] == 'miss'
# "5000" is an invalid id; but by default cache_missing_ids is set to true, so the
# empty response would've been cached from the above request
hit_response = assert_is_in_spectre_cache(path.format(ids='5000'))
assert hit_response.json() == []
def test_dont_drop_underscored_headers(self):
response = get_through_spectre(
'/business?foo=bar&business_id=1234',
extra_headers={
'Test-Header': 'val1',
'Header_with_underscores': 'val2',
},
)
headers = response.json()['received_headers']
assert 'test-header' in headers
assert 'header_with_underscores' in headers
def test_with_invalid_id_when_cache_missing_ids_is_true(self):
base_path = '/bulk_requester_2'
# 0 < ids < 1000 are valid
resp1 = get_through_spectre("{}/{}/v1?k1=v2".format(base_path, '10,5000,11'))
headers1, body1 = resp1.headers, resp1.json()
assert len(body1) == 2
assert headers1['Spectre-Cache-Status'] == 'miss'
# "5000" is an invalid id; but by default cache_missing_ids is set to true, so the
# empty response would've been cached from the above request
resp2 = assert_is_in_spectre_cache("{}/{}/v1?k1=v2".format(base_path, '5000'))
resp2.json() == []
def test_purge_cassandra_by_id_backward_compatible(self):
# Delete after PERF-2453 is done
get_through_spectre('/bulk_requester?ids=1')
assert_is_in_spectre_cache('/bulk_requester?ids=1')
# Purge id 1
res = requests.request(
'PURGE',
util.SPECTRE_BASE_URL +
'?cache_name=bulk_requester_does_not_cache_missing_ids&id=1',
headers=util.HAPROXY_ADDED_HEADERS)
assert res.status_code == 200
get_resp_3 = get_through_spectre('/bulk_requester?ids=1')
# resp 3 was no longer cached.
assert get_resp_3.headers['Spectre-Cache-Status'] == 'miss'
res = requests.request(
'PURGE',
util.SPECTRE_BASE_URL +
'?cache_name=bulk_requester_does_not_cache_missing_ids',
headers=util.HAPROXY_ADDED_HEADERS)
assert res.status_code == 200
def test_bulk_ids_in_path(self):
base_path = '/bulk_requester_2'
resp_1 = get_through_spectre("{}/{}/v1?k1=v2".format(base_path, '1,2,3'))
# Bulk was a miss
bulk_headers, bulk_body = resp_1.headers, resp_1.json()
assert bulk_headers['Spectre-Cache-Status'] == 'miss'
resp_2 = assert_is_in_spectre_cache("{}/{}/v1?k1=v2".format(base_path, '2'))
_, body = resp_2.headers, resp_2.json()
# Check for correctness
assert len(body) == 1
assert bulk_body[1] == body[0]
def test_cassandra_timeouts_dont_slow_down_spectre(self):
# Run the test twice to make sure the slow write from the first request
# doesn't affect the next request.
for i in range(2):
start = time.time()
response = get_through_spectre('/timestamp/get')
duration = time.time() - start
# Cassandra timeouts are treated as a miss.
assert response.headers['Spectre-Cache-Status'] == 'miss'
# The timeouts are set at 500ms in itests.
# Duration should be very close to that value. If it's much higher it
# means the timeouts are not working, if it's lower it means the
# iptable rule is not working.
assert duration - 0.5 < 0.01
def make_request(self, ids, cache=True, extra_headers=None, assert_hit=False):
base_path = '/bulk_requester'
ids = [str(i) for i in ids]
ids = "%2C".join(ids)
if not cache:
cache_headers = {'Cache-Control': 'no-cache'}
if extra_headers is None:
extra_headers = {}
extra_headers.update(cache_headers)
if assert_hit:
resp = assert_is_in_spectre_cache("{}?ids={}".format(base_path, ids), extra_headers)
else:
resp = get_through_spectre("{}?ids={}".format(base_path, ids), extra_headers)
return resp.headers, resp.json()
def test_can_skip_cassandra_check(self):
response = get_through_spectre('/status')
assert response.status_code == 200
assert response.text == 'Backend is alive\n'
response = get_from_spectre('/status')
assert response.status_code == 200
status = json.loads(response.text)
assert status['cassandra_status'] == 'skipped'
assert status['smartstack_configs'] == 'present'
assert status['spectre_configs'] == 'present'
assert status['proxied_services'] == {
'backend.main': {
'host': '10.5.0.3',
'port': 9080,
},
}
def _call_with_zipkin(
self,
trace_id,
span_id,
parent_span_id,
sampled='1',
url='/not_cacheable',
):
"""Calls Spectre-fronted service with specified Zipkin HTTP headers."""
zipkin_headers = {
'X-B3-TraceId': trace_id,
'X-B3-SpanId': span_id,
'X-B3-ParentSpanId': parent_span_id,
'X-B3-Flags': '0',
'X-B3-Sampled': sampled,
}
response = get_through_spectre(
url,
extra_headers=zipkin_headers,
)
return response
def test_bulk_endpoint_miss(log_file):
response = get_through_spectre(
'/bulk_requester_2/10,11/v1?foo=bar',
)
assert response.status_code == 200
assert response.headers['Spectre-Cache-Status'] == 'miss'
metrics = _load_metrics(log_file)
# We have `spectre.fetch_body_and_headers` and `spectre.hit_rate` twice
# since we have 2 ids in the url.
_assert_fetch_hit_rate(metrics[0:2], 'bulk_requester_default')
_assert_fetch_hit_rate(metrics[2:4], 'bulk_requester_default')
# Then we have 2 `spectre.store_body_and_headers`
_assert_store_metric(metrics[4], 'bulk_requester_default')
_assert_store_metric(metrics[5], 'bulk_requester_default')
# Then the `spectre.request_timing`
_assert_request_timing_metrics(metrics[6:10], 'bulk_requester_default')
# Finally we have `spectre.bulk_hit_rate`
assert metrics[10].dimensions == {
'metric_name': 'spectre.bulk_hit_rate',
'habitat': 'uswest1a',
'service_name': 'spectre',
'namespace': 'backend.main',
'instance_name': 'itest',
'cache_name': 'bulk_requester_default',
'cache_status': 'miss',
}
# This assert is mostly there to make sure there are no more metrics than what I expect.
# The reason why it's not before the other asserts is because the error message doesn't
# show you what metrics are actually in the list, so it's very annoying to figure out
# what's missing. You'd need to comment out this check and then verify which of the
# other asserts is failing.
assert len(metrics) == 11
def test_bulk_request_error_codes(self):
resp = get_through_spectre('/bulk_requester_2/1/v1?error_status=502')
assert resp.status_code == 502
assert resp.headers[
'Spectre-Cache-Status'] == 'non-cacheable-response: status code is 502'
def test_doesnt_validate_json_on_non_200(self):
resp = get_through_spectre("/not_authorized?ids=1")
assert resp.status_code == 403
assert resp.headers[
'Spectre-Cache-Status'] == 'non-cacheable-response: status code is 403'
def test_bulk_endpoint_regex_can_work(self, url, cache_status):
assert get_through_spectre(
url).headers['Spectre-Cache-Status'] == cache_status
