def test_if_early_boot_stage_can_recover_from_a_bit_slow_backend(
self, nginx_class, valid_user_header, mocker, log_catcher):
# The idea here is to make Backend a bit slow, so that AR is still able
# to update cache on first request.
refresh_lock_timeout = 10
backend_request_timeout = 5
ar = nginx_class(
cache_first_poll_delay=1,
cache_poll_period=3,
cache_expiration=2,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
cache_backend_request_timeout=backend_request_timeout,
cache_refresh_lock_timeout=refresh_lock_timeout,
)
agent_id = AGENT1_ID
url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
v = Vegeta(log_catcher, target=url, jwt=valid_user_header, rate=3)
# Make mesos just a bit :)
# It mus respond slower than backend_request_timeout
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_stall',
aux_data=backend_request_timeout * 0.3)
with GuardedSubprocess(ar):
with GuardedSubprocess(v):
time.sleep(backend_request_timeout * 0.3 +
1) # let it warm-up!
ping_mesos_agent(ar, valid_user_header)
def test_if_broken_mesos_does_not_prevent_resolving_via_marathon(
self, mocker, nginx_class, valid_user_header):
# Bork Mesos Mock, Make MesosDNS mock respond with no apps, so that AR
# is able to resolve only via Marathon/we are certain that it resolved
# via Marathon.
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=EMPTY_SRV)
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(
cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
)
url = ar.make_url_from_path("/service/scheduler-alwaysthere/")
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
data = resp.json()
assert data['endpoint_id'] == 'http://127.0.0.1:16000'
def test_ip_per_task_app_with_unspecified_ip_address_DCOS_OSS_1366(
self, nginx_class, mocker, valid_user_header):
"""
Test that an app that, instead of specifying 'ipAddress: null' does not
specify 'ipAddress' at all, is successfully cached.
"""
app = self._scheduler_alwaysthere_app()
# Remove the 'ipAddress' key completely, thereby triggering DCOS_OSS-1366.
del (app["ipAddress"])
ar = nginx_class()
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": [app]})
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.1:16000'
def test_ip_per_task_app_without_user_networking(self, nginx_class, mocker,
valid_user_header):
app = self._scheduler_alwaysthere_app()
app['ipAddress'] = {
'networkName': 'samplenet',
'discovery': {
"ports": [{
"number": 80,
"name": "http",
"protocol": "tcp"
}]
}
}
app['tasks'][0]['ipAddresses'][0]['ipAddress'] = '127.0.0.2'
ar = nginx_class()
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": [app]})
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.2:80'
def test_mesos_leader_reresolve_in_proxy_pass(
self,
nginx_class,
valid_user_header,
dns_server_mock,
path,
dest_port,
):
# Change the TTL of `leader.mesos.` entry
dns_server_mock.set_dns_entry(
'leader.mesos.', ip='127.0.0.2', ttl=self.LONG_TTL)
ar = nginx_class(upstream_mesos="http://leader.mesos:5050")
with GuardedSubprocess(ar):
generic_correct_upstream_dest_test(
ar,
valid_user_header,
path,
"http://127.0.0.2:{}".format(dest_port),
)
# Update the `leader.mesos.` entry with new value
dns_server_mock.set_dns_entry(
'leader.mesos.', ip='127.0.0.3', ttl=self.LONG_TTL)
# This should be equal to 1.5 times the value of `valid=` DNS TTL
# override in `resolver` config option -> 5s * 1.5 = 7.5s
time.sleep(5 * 1.5)
generic_correct_upstream_dest_test(
ar,
valid_user_header,
path,
"http://127.0.0.3:{}".format(dest_port),
)
def test_if_default_scheme_is_honoured_by_agent_endpoint(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}
ar = nginx_class(default_scheme="https://")
agent_id = AGENT3_ID
url_good = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
agent_id = AGENT1_ID
url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url_bad,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 502
resp = requests.get(url_good,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'https://127.0.0.1:15401'
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_default_scheme_is_honoured_by_agent_endpoint(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}
ar = nginx_class(default_scheme="https://")
agent_id = '35f210bb-bb58-4559-9932-b62619e72b6d-S0'
url_good = ar.make_url_from_path(
'/agent/{}/blah/blah'.format(agent_id))
agent_id = 'de1baf83-c36c-4d23-9cb0-f89f596cd6ab-S1'
url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url_bad,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 502
resp = requests.get(url_good,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'https://127.0.0.1:15401'
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_mesos_does_not_break_marathon_cache(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(1, True),
}
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
ar = nginx_class()
url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.1:16000'
assert lbf.extra_matches == {}
def test_if_caching_works_for_marathon_apps(self, nginx_class, mocker,
valid_user_header):
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
ar = nginx_class()
url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz')
with GuardedSubprocess(ar):
# Let the cache warm-up:
time.sleep(CACHE_FIRST_POLL_DELAY + 1)
for _ in range(5):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
mesos_requests = mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
# 3 requests + only one upstream requst == cache works
assert len(mesos_requests) == 1
assert len(marathon_requests) == 2
def test_if_broken_marathon_does_not_prevent_resolving_root_marathon(
self, mocker, nginx_class, valid_user_header):
# Bork Marathon Mock, DO NOT touch Mesos Mock:
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Change the Root Marathon's endpoint address as reported by Mesos mock,
# so that we do not get the reply from the original endpoint (i.e.
# http://127.0.0.1:8080 as it will always respond with broken responses
# (see `mocker.send_command` call above).
fwrk = framework_from_template(SCHEDULER_FWRK_ALWAYSTHERE_ID,
"marathon", "http://127.0.0.2:8080/")
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[fwrk])
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(
cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
)
url = ar.make_url_from_path("/service/marathon/v2/reflect/me")
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
data = resp.json()
assert data['endpoint_id'] == 'http://127.0.0.2:8080'
def test_if_broken_marathon_does_not_prevent_resolving_root_metronome(
self, mocker, nginx_class, valid_user_header):
# Bork Marathon Mock, DO NOT touch Mesos Mock:
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(
cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
)
url = ar.make_url_from_path("/service/metronome/foo/bar")
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
data = resp.json()
assert data['endpoint_id'] == 'http://127.0.0.1:9000'
def test_upstream_wrong_json(self, nginx_class, mocker, valid_user_header):
filter_regexp = {
"Cannot decode Marathon apps JSON: ": SearchCriteria(1, True),
}
ar = nginx_class()
# Set wrong non-json response content
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_encoded_response',
aux_data=b"wrong response")
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert "cache state is invalid" in resp.content.decode('utf-8')
assert resp.status_code == 503
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_missing_mesos_leader_entry_is_handled(self, nginx_class,
valid_user_header,
dns_server_mock):
filter_regexp = {
'Failed to instantiate the resolver':
SearchCriteria(0, True),
'DNS server returned error code':
SearchCriteria(1, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(0, True),
}
ar = nginx_class()
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
# Unfortunatelly there are upstreams that use `leader.mesos` and
# removing this entry too early will result in Nginx failing to start.
# So we need to do it right after nginx starts, but before first
# cache update.
time.sleep(1)
dns_server_mock.remove_dns_entry('leader.mesos.')
# Now let's trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_caching_works_for_marathon_leader(self, nginx_class, mocker,
valid_user_header):
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
ar = nginx_class()
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
# Let the cache warm-up:
time.sleep(CACHE_FIRST_POLL_DELAY + 1)
for _ in range(5):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.2:80'
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
# 3 requests + only one upstream request == cache works
assert len(marathon_requests) == 2
def test_if_changing_marathon_leader_is_reflected_in_cache(
self, nginx_class, mocker, valid_user_header):
cache_poll_period = 4
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
# let's make sure that current leader is the default one
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.2:80'
# change the leader and wait for cache to notice
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='change_leader',
aux_data="127.0.0.3:80")
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
# now, let's see if the leader changed
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.3:80'
def test_if_broken_mesos_prevents_resolving_via_mesosdns(
self, mocker, nginx_class, valid_user_header):
# Bork Mesos Mock, Make Marathon mock respond with no apps, so that AR
# tries to resolve via Mesos /state-summary
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": []})
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(
cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
)
url = ar.make_url_from_path("/service/scheduler-alwaysthere/")
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 503
assert '503 Service Unavailable: invalid Mesos state cache' == resp.text.strip(
)
def test_if_changing_marathon_apps_is_reflected_in_cache(
self, nginx_class, valid_user_header, mocker):
cache_poll_period = 4
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz')
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.1:16000'
new_apps = {
"apps": [
SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT,
]
}
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data=new_apps)
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.15:16001'
def test_if_broken_marathon_prevents_resolving_via_mesos_state_summary(
self, mocker, nginx_class, valid_user_header):
# Bork Marathon Mock, DO NOT touch Mesos Mock:
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(
cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
)
url = ar.make_url_from_path("/service/scheduler-alwaysthere/")
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 503
assert '503 Service Unavailable: invalid Marathon svcapps cache' in resp.text
def test_if_broken_marathon_does_not_break_mesos_cache(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
}
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
ar = nginx_class()
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_changing_marathon_apps_is_reflected_in_cache(
self, nginx_class, superuser_user_header, mocker):
cache_poll_period = 4
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
url = ar.make_url_from_path('/service/nginx-enabled/bar/baz')
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 500
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='enable_nginx_app')
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.1:16001'
def test_if_mesos_leader_locality_is_resolved(self, nginx_class,
valid_user_header,
dns_server_mock):
cache_poll_period = 4
nonlocal_leader_ip = "127.0.0.3"
local_leader_ip = "127.0.0.2"
filter_regexp_pre = {
'Failed to instantiate the resolver':
SearchCriteria(0, True),
'Mesos Leader is non-local: `{}`'.format(nonlocal_leader_ip):
SearchCriteria(1, True),
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(0, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
filter_regexp_post = {
'Failed to instantiate the resolver':
SearchCriteria(0, True),
'Mesos Leader is local':
SearchCriteria(1, True),
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(0, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip)
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp_pre,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip)
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
lbf = LineBufferFilter(filter_regexp_post,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_absence_of_agent_is_handled_by_cache(self, nginx_class, mocker,
valid_user_header):
ar = nginx_class()
with GuardedSubprocess(ar):
ping_mesos_agent(
ar,
valid_user_header,
agent_id='bdcd424a-b59e-4df4-b492-b54e38926bd8-S0',
expect_status=404)
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, False),
'Mesos state cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `svcapps` is too old, aborting request':
SearchCriteria(1, True),
}
ar = nginx_class(
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=4,
cache_expiration=2,
cache_poll_period=3,
)
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='enable_nginx_app')
url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be discarded by AR:
# cache_max_age_hard_limit + 1s for good measure
# must be more than cache_poll_period
time.sleep(4 + 1)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 503
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_leader_is_unknown_state_is_handled(
self, nginx_class, valid_user_header):
ar = nginx_class(host_ip=None)
url = ar.make_url_from_path('/dcos-history-service/foo/bar')
with GuardedSubprocess(ar):
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 503
assert 'mesos leader is unknown' in resp.text
def test_if_historyservice_endpoint_auth_precedence_is_enforced(
self, valid_user_header, mocker, nginx_class):
ar = nginx_class(host_ip=None)
url = ar.make_url_from_path('/dcos-history-service/foo/bar')
with GuardedSubprocess(ar):
resp = requests.get(url, allow_redirects=False)
assert resp.status_code == 401
resp = requests.get(url, allow_redirects=False, headers=valid_user_header)
assert resp.status_code == 503
def test_if_not_defining_the_var_is_handled(self, nginx_class, role):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = {'SECRET_KEY_FILE_PATH not set.': SearchCriteria(1, False)}
ar = nginx_class(role=role, secret_key_file_path=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_ar_with_empty_cache_waits_for_marathon_during_service_resolve(
self, mocker, nginx_class, valid_user_header):
# Make service endpoint resolve only Marathon-related data:
mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(
endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=EMPTY_SRV)
# Make Mock endpoint stall a little, make sure AR cache update timeouts
# are big enough to swallow it:
backend_request_timeout = 6
refresh_lock_timeout = backend_request_timeout * 2
# Make period cache refreshes so rare that they do not get into
# picture:
ar = nginx_class(cache_first_poll_delay=1200,
cache_poll_period=1200,
cache_expiration=1200,
cache_max_age_soft_limit=1200,
cache_max_age_hard_limit=1800,
cache_backend_request_timeout=backend_request_timeout,
cache_refresh_lock_timeout=refresh_lock_timeout,
)
url = ar.make_url_from_path("/service/scheduler-alwaysthere/")
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_stall',
aux_data=backend_request_timeout * 0.5)
# Measure the time it took and the results:
with GuardedSubprocess(ar):
t_start = time.time()
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
t_spent = time.time() - t_start
assert resp.status_code == 200
data = resp.json()
assert data['endpoint_id'] == 'http://127.0.0.1:16000'
# If AR waits for cache during resolve, then time spent should be
# greater than the stall time that has been set. Due to the fact
# that update coroutines are not separated yet, this will be
# slightly higher than: 2 * (backend_request_timeout * 0.5)
# as we have two calls to Marathon (svcapps + marathon leader) from
# the cache code.
assert t_spent > 2 * (backend_request_timeout * 0.5)
def test_if_cache_refresh_occurs_regularly(self, nginx_class, mocker,
valid_user_header):
filter_regexp = {
'Executing cache refresh triggered by timer':
SearchCriteria(3, False),
'Cache `[\s\w]+` expired. Refresh.':
SearchCriteria(8, True),
'Mesos state cache has been successfully updated':
SearchCriteria(3, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(3, True),
'Marathon leader cache has been successfully updated':
SearchCriteria(3, True),
}
cache_poll_period = 4
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make regular polling occur faster than usual to speed up the tests.
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
# In total, we should get three cache updates in given time frame:
timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=timeout,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
# Do a request that uses cache so that we can verify that data was
# in fact cached and no more than one req to mesos/marathon
# backends were made
ping_mesos_agent(ar, valid_user_header)
mesos_requests = mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 3
assert len(marathon_requests) == 6
def _assert_filter_regexp_for_invalid_app(
self,
filter_regexp,
app,
nginx_class,
mocker,
auth_headers,
):
"""Helper method that will assert if provided regexp filter is found
in nginx logs for given apps response from Marathon upstream endpoint.
Arguments:
filter_regexp (dict): Filter definition where key is the message
looked up in logs and value is SearchCriteria definition
app (dict): App that upstream endpoint should respond with
nginx_class (Nginx): Nginx process fixture
mocker (Mocker): Mocker fixture
auth_header (dict): Headers that should be passed to Nginx in the
request
"""
ar = nginx_class()
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": [app]})
# Remove all entries for mesos frameworks and mesos_dns so that
# we test only the information in Marathon
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=[])
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=auth_headers)
assert resp.status_code == 500
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True),
}
ar = nginx_class(host_ip=valid_ip)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
