def test_if_broken_marathon_does_not_break_mesos_cache(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
}
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
ar = nginx_class()
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
ping_mesos_agent(ar, superuser_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_default_scheme_is_honoured_by_agent_endpoint(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}
ar = nginx_class(default_scheme="https://")
agent_id = AGENT3_ID
url_good = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
agent_id = AGENT1_ID
url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url_bad,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 502
resp = requests.get(url_good,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'https://127.0.0.1:15401'
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_upstream_wrong_json(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
"Cannot decode Marathon apps JSON: ": SearchCriteria(1, True),
}
ar = nginx_class()
# Set wrong non-json response content
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_encoded_response',
aux_data=b"wrong response")
url = ar.make_url_from_path('/service/nginx-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert "cache state is invalid" in resp.content.decode('utf-8')
assert resp.status_code == 503
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_mesos_does_not_break_marathon_cache(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated': SearchCriteria(1, True),
}
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='enable_nginx_app')
ar = nginx_class()
url = ar.make_url_from_path('/service/nginx-enabled/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'http://127.0.0.1:16001'
assert lbf.extra_matches == {}
def test_if_missing_mesos_leader_entry_is_handled(
self, nginx_class, valid_user_header, dns_server_mock):
filter_regexp = {
'Failed to instantiate the resolver': SearchCriteria(0, True),
'DNS server returned error code': SearchCriteria(1, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(0, True),
}
ar = nginx_class()
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
# Unfortunatelly there are upstreams that use `leader.mesos` and
# removing this entry too early will result in Nginx failing to start.
# So we need to do it right after nginx starts, but before first
# cache update.
time.sleep(1)
dns_server_mock.remove_dns_entry('leader.mesos.')
# Now let's trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, False),
'Mesos state cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `svcapps` is too old, aborting request':
SearchCriteria(1, True),
}
ar = nginx_class(
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=4,
cache_expiration=2,
cache_poll_period=3,
)
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='enable_nginx_app')
url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be discarded by AR:
# cache_max_age_hard_limit + 1s for good measure
# must be more than cache_poll_period
time.sleep(4 + 1)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 503
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_pointing_to_empty_file_is_handled(self,
ar_process_without_secret_key):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = 'Secret key not set or empty string.'
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar_process_without_secret_key.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.all_found is True
def test_if_not_defining_the_var_is_handled(self,
ar_process_without_secret_key):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = 'SECRET_KEY_FILE_PATH not set.'
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar_process_without_secret_key.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.all_found is True
def test_if_not_defining_the_var_is_handled(self, nginx_class, role):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = {'SECRET_KEY_FILE_PATH not set.': SearchCriteria(1, False)}
ar = nginx_class(role=role, secret_key_file_path=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def _assert_filter_regexp_for_invalid_app(
self,
filter_regexp,
app,
nginx_class,
mocker,
auth_headers,
):
"""Helper method that will assert if provided regexp filter is found
in nginx logs for given apps response from Marathon upstream endpoint.
Arguments:
filter_regexp (dict): Filter definition where key is the message
looked up in logs and value is SearchCriteria definition
app (dict): App that upstream endpoint should respond with
nginx_class (Nginx): Nginx process fixture
mocker (Mocker): Mocker fixture
auth_header (dict): Headers that should be passed to Nginx in the
request
"""
ar = nginx_class()
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": [app]})
# Remove all entries for mesos frameworks and mesos_dns so that
# we test only the information in Marathon
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=[])
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=auth_headers)
assert resp.status_code == 500
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, False),
'Mesos state cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `svcapps` is too old, aborting request':
SearchCriteria(1, True),
}
ar = nginx_class(cache_max_age_soft_limit=3,
cache_max_age_hard_limit=4,
cache_expiration=2,
cache_poll_period=3,
)
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='enable_nginx_app')
url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be discarded by AR:
# cache_max_age_hard_limit + 1s for good measure
# must be more than cache_poll_period
time.sleep(4 + 1)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 503
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly(self, nginx_class, mocker,
valid_user_header):
filter_regexp = {
'Executing cache refresh triggered by timer':
SearchCriteria(3, False),
'Cache `[\s\w]+` expired. Refresh.':
SearchCriteria(8, True),
'Mesos state cache has been successfully updated':
SearchCriteria(3, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(3, True),
'Marathon leader cache has been successfully updated':
SearchCriteria(3, True),
}
cache_poll_period = 4
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make regular polling occur faster than usual to speed up the tests.
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
# In total, we should get three cache updates in given time frame:
timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=timeout,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
# Do a request that uses cache so that we can verify that data was
# in fact cached and no more than one req to mesos/marathon
# backends were made
ping_mesos_agent(ar, valid_user_header)
mesos_requests = mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 3
assert len(marathon_requests) == 6
def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True),
}
ar = nginx_class(host_ip=valid_ip)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_absent_var_is_handled(self, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: unknown': SearchCriteria(1, True),
}
ar = nginx_class(host_ip=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True),
}
ar = nginx_class(host_ip=valid_ip)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_not_defining_the_var_is_handled(self,
ar_process_without_secret_key):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = 'SECRET_KEY_FILE_PATH not set.'
lbf = LineBufferFilter(
filter_regexp,
line_buffer=ar_process_without_secret_key.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.all_found is True
def test_if_absent_var_is_handled(self, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: unknown': SearchCriteria(1, True),
}
ar = nginx_class(host_ip=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_pointing_to_empty_file_is_handled(
self, ar_process_without_secret_key):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = 'Secret key not set or empty string.'
lbf = LineBufferFilter(
filter_regexp,
line_buffer=ar_process_without_secret_key.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.all_found is True
def _assert_filter_regexp_for_invalid_app(
self,
filter_regexp,
app,
nginx_class,
mocker,
auth_headers,
):
"""Helper method that will assert if provided regexp filter is found
in nginx logs for given apps response from Marathon upstream endpoint.
Arguments:
filter_regexp (dict): Filter definition where key is the message
looked up in logs and value is SearchCriteria definition
app (dict): App that upstream endpoint should respond with
nginx_class (Nginx): Nginx process fixture
mocker (Mocker): Mocker fixture
auth_header (dict): Headers that should be passed to Nginx in the
request
"""
ar = nginx_class()
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data={"apps": [app]})
# Remove all entries for mesos frameworks and mesos_dns so that
# we test only the information in Marathon
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=[])
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=auth_headers)
assert resp.status_code == 404
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_temp_marathon_borkage_does_not_disrupt_caching(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, False),
'Mesos state cache has been successfully updated':
SearchCriteria(2, False),
'Using stale `svcapps` cache entry to fulfill the request':
SearchCriteria(1, True),
}
ar = nginx_class(
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=1200,
cache_expiration=2,
cache_poll_period=3,
)
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be considered stale by AR:
# cache_max_age_soft_limit + 1s for a good measure
time.sleep(3 + 1)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_cache_refresh_is_triggered_by_request(self, nginx_class,
mocker,
valid_user_header):
"""...right after Nginx has started."""
filter_regexp = {
'Executing cache refresh triggered by request':
SearchCriteria(1, True),
'Cache `[\s\w]+` empty. Fetching.':
SearchCriteria(3, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(1, True),
'Marathon leader cache has been successfully updated':
SearchCriteria(1, True),
}
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make sure that timers will not interfere:
ar = nginx_class(cache_first_poll_delay=120,
cache_poll_period=120,
cache_expiration=115)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=5,
line_buffer=ar.stderr_line_buffer)
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
# Do an extra request so that we can verify that data was in fact
# cached and no more than one req to mesos/marathon backends were
# made
ping_mesos_agent(ar, valid_user_header)
mesos_requests = mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 1
assert len(marathon_requests) == 2
def test_if_var_pointing_to_empty_file_is_handled(
self, nginx_class, role, empty_file):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = {'Auth token verification key not set': SearchCriteria(1, False)}
ar = nginx_class(role=role, auth_token_verification_key_file_path=empty_file)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_pointing_to_empty_file_is_handled(
self, nginx_class, role, empty_file):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = {'Auth token verification key not set': SearchCriteria(1, False)}
ar = nginx_class(role=role, auth_token_verification_key_file_path=empty_file)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_temp_marathon_borkage_does_not_disrupt_caching(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Marathon app request failed: invalid response status: 500':
SearchCriteria(1, False),
'Mesos state cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `svcapps` is stale':
SearchCriteria(1, True),
}
ar = nginx_class(cache_max_age_soft_limit=3,
cache_max_age_hard_limit=1200,
cache_expiration=2,
cache_poll_period=3,
)
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
# Break marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be considered stale by AR:
# cache_max_age_soft_limit + 1s for a good measure
time.sleep(3 + 1)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_not_defining_the_var_is_handled(self, nginx_class, role):
# Scanning for the exact log entry is bad, but in this case - can't be
# avoided.
filter_regexp = {
'AUTH_TOKEN_VERIFICATION_KEY_FILE_PATH not set.':
SearchCriteria(1, False)
}
ar = nginx_class(role=role, auth_token_verification_key_file_path=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_is_verified(self, invalid_ip, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: unknown': SearchCriteria(1, True),
'HOST_IP var is not a valid ipv4: {}'.format(invalid_ip):
SearchCriteria(1, True),
}
ar = nginx_class(host_ip=invalid_ip)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_var_is_verified(self, invalid_ip, nginx_class, mocker):
filter_regexp = {
'Local Mesos Master IP: unknown': SearchCriteria(1, True),
'HOST_IP var is not a valid ipv4: {}'.format(invalid_ip):
SearchCriteria(1, True),
}
ar = nginx_class(host_ip=invalid_ip)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_public_ip_detect_script_failue_is_handled(
self, master_ar_process_perclass, valid_user_header):
url = master_ar_process_perclass.make_url_from_path('/metadata')
filter_regexp = {
'Traceback \(most recent call last\):':
SearchCriteria(1, True),
("FileNotFoundError: \[Errno 2\] No such file or directory:"
" '/usr/local/detect_ip_public_data.txt'"):
SearchCriteria(1, True),
}
lbf = LineBufferFilter(
filter_regexp,
line_buffer=master_ar_process_perclass.stderr_line_buffer)
with lbf, overridden_file_content(
'/usr/local/detect_ip_public_data.txt'):
os.unlink('/usr/local/detect_ip_public_data.txt')
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
assert lbf.extra_matches == {}
resp_data = resp.json()
assert resp_data['PUBLIC_IPV4'] == "127.0.0.1"
def test_if_valid_auth_attempt_is_logged_correctly(
self, master_ar_process, valid_jwt_generator, mocker):
# Create some random, unique user that we can grep for:
uid = 'some_random_string_abc213421341'
mocker.send_command(endpoint_id='http://127.0.0.1:8101',
func_name='add_user',
aux_data={'uid': uid})
filter_regexp = {
'validate_jwt_or_exit\(\): UID from valid JWT: `{}`'.format(uid):
SearchCriteria(1, False)}
lbf = LineBufferFilter(filter_regexp,
line_buffer=master_ar_process.stderr_line_buffer)
# Create token for this user:
token = valid_jwt_generator(uid)
header = {'Authorization': 'token={}'.format(token)}
url = master_ar_process.make_url_from_path()
with lbf:
resp = requests.get(url,
allow_redirects=False,
headers=header)
assert resp.status_code == 200
assert lbf.extra_matches == {}
def test_if_first_cache_refresh_occurs_earlier(self, nginx_class, mocker,
valid_user_header):
filter_regexp = {
'Executing cache refresh triggered by timer':
SearchCriteria(1, False),
'Cache `[\s\w]+` empty. Fetching.':
SearchCriteria(3, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(1, True),
'Marathon leader cache has been successfully updated':
SearchCriteria(1, True),
}
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for Mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make regular polling occur later than usual, so that we get clear
# results.
ar = nginx_class(cache_poll_period=60, cache_expiration=55)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
# Do a request that uses cache so that we can verify that data was
# in fact cached and no more than one req to mesos/marathon
# backends were made
ping_mesos_agent(ar, valid_user_header)
mesos_requests = mocker.send_command(
endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(
endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 1
assert len(marathon_requests) == 2
def test_if_mesos_leader_locality_is_resolved(self, nginx_class,
valid_user_header,
dns_server_mock):
cache_poll_period = 4
nonlocal_leader_ip = "127.0.0.3"
local_leader_ip = "127.0.0.2"
filter_regexp_pre = {
'Failed to instantiate the resolver':
SearchCriteria(0, True),
'Mesos Leader is non-local: `{}`'.format(nonlocal_leader_ip):
SearchCriteria(1, True),
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(0, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
filter_regexp_post = {
'Failed to instantiate the resolver':
SearchCriteria(0, True),
'Mesos Leader is local':
SearchCriteria(1, True),
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(0, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip)
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=3)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp_pre,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip)
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
lbf = LineBufferFilter(filter_regexp_post,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header):
filter_regexp = {
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(1, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
ar = nginx_class(host_ip=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_auth_module_is_enabled_by_unless_false_str_is_provided(
self, nginx_class, mocker, enable_keyword):
filter_regexp = {
'Activate authentication module.': SearchCriteria(1, True),
}
ar = nginx_class(auth_enabled=enable_keyword)
url = ar.make_url_from_path('/exhibitor/foo/bar')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url, allow_redirects=False)
assert resp.status_code == 401
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header):
filter_regexp = {
'Local Mesos Master IP address is unknown, cache entry is unusable':
SearchCriteria(1, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
ar = nginx_class(host_ip=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, False),
'Marathon apps cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `mesosstate` is too old, aborting request':
SearchCriteria(1, True),
}
ar = nginx_class(
cache_poll_period=3,
cache_expiration=2,
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=4,
)
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update using a request:
ping_mesos_agent(ar, valid_user_header)
# Break mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be discarded by AR:
# cache_max_age_hard_limit + 1s for good measure
# must be more than cache_poll_period
time.sleep(4 + 1)
# Perform the main/test request:
ping_mesos_agent(ar, valid_user_header, expect_status=503)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Executing cache refresh triggered by timer': SearchCriteria(3, False),
'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(6, True),
'Mesos state cache has been successfully updated': SearchCriteria(3, True),
'Marathon apps cache has been successfully updated': SearchCriteria(3, True),
'Marathon leader cache has been successfully updated': SearchCriteria(3, True),
}
cache_poll_period = 4
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make regular polling occur faster than usual to speed up the tests.
ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3)
# In total, we should get three cache updates in given time frame:
timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=timeout,
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
# Do a request that uses cache so that we can verify that data was
# in fact cached and no more than one req to mesos/marathon
# backends were made
ping_mesos_agent(ar, superuser_user_header)
mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 3
assert len(marathon_requests) == 6
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header):
filter_regexp = {
'Private IP address of the host is unknown, ' +
'aborting cache-entry creation for mesos leader':
SearchCriteria(1, True),
'mesos leader cache has been successfully updated':
SearchCriteria(1, True),
}
ar = nginx_class(host_ip=None)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_auth_module_can_be_disabled(self, nginx_class, mocker):
filter_regexp = {
("ADMINROUTER_ACTIVATE_AUTH_MODULE set to `false`. "
"Deactivate authentication module."): SearchCriteria(1, True),
}
ar = nginx_class(auth_enabled='false')
url = ar.make_url_from_path('/exhibitor/foo/bar')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False)
assert resp.status_code == 200
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_temp_mesos_borkage_does_not_dirupt_caching(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, False),
'Marathon apps cache has been successfully updated':
SearchCriteria(2, False),
'Using stale `mesosstate` cache entry to fulfill the request':
SearchCriteria(1, True),
}
ar = nginx_class(
cache_poll_period=3,
cache_expiration=2,
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=1800,
)
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(
filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update using a request:
ping_mesos_agent(ar, valid_user_header)
# Break mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to become stale:
# cache_max_age_soft_limit + 1s for good measure
time.sleep(3 + 1)
# Perform the main/test request:
ping_mesos_agent(ar, valid_user_header, expect_status=200)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_auth_module_is_enabled_by_unless_false_str_is_provided(
self, nginx_class, mocker, enable_keyword):
filter_regexp = {
'Activate authentication module.': SearchCriteria(1, True),
}
ar = nginx_class(auth_enabled=enable_keyword)
url = ar.make_url_from_path('/exhibitor/foo/bar')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False)
assert resp.status_code == 401
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_cache_refresh_is_triggered_by_request(
self, nginx_class, mocker, superuser_user_header):
"""...right after Nginx has started."""
filter_regexp = {
'Executing cache refresh triggered by request': SearchCriteria(1, True),
'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True),
'Mesos state cache has been successfully updated': SearchCriteria(1, True),
'Marathon apps cache has been successfully updated': SearchCriteria(1, True),
'Marathon leader cache has been successfully updated': SearchCriteria(1, True),
}
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make sure that timers will not interfere:
ar = nginx_class(cache_first_poll_delay=120,
cache_poll_period=120,
cache_expiration=115)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=5,
line_buffer=ar.stderr_line_buffer)
ping_mesos_agent(ar, superuser_user_header)
lbf.scan_log_buffer()
# Do an extra request so that we can verify that data was in fact
# cached and no more than one req to mesos/marathon backends were
# made
ping_mesos_agent(ar, superuser_user_header)
mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 1
assert len(marathon_requests) == 2
def test_if_auth_module_can_be_disabled(self, nginx_class, mocker):
filter_regexp = {
("ADMINROUTER_ACTIVATE_AUTH_MODULE set to `false`. "
"Deactivate authentication module."): SearchCriteria(1, True),
}
ar = nginx_class(auth_enabled='false')
url = ar.make_url_from_path('/exhibitor/foo/bar')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False)
assert resp.status_code == 200
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def assert_endpoint_response(ar,
path,
code,
assert_error_log=None,
headers=None,
cookies=None,
assertions=None):
"""Asserts response code and log messages in Admin Router stderr for
request against specified path.
Arguments:
ar (Nginx): Running instance of the AR
code (int): Expected response code
assert_error_log (dict): LineBufferFilter compatible definition of messages
to assert
cookies (dict): Optionally provide request cookies
headers (dict): Optionally provide request headers
assertions (List[lambda r]) Optionally provide additional assertions
for the response
"""
def body():
r = requests.get(
ar.make_url_from_path(path),
headers=headers,
cookies=cookies,
)
assert r.status_code == code
if assertions:
for func in assertions:
assert func(r)
if assert_error_log is not None:
# for testing, log messages go to both stderr and to /dev/log
with LineBufferFilter(copy.deepcopy(assert_error_log),
line_buffer=ar.stderr_line_buffer) as stderr:
with LineBufferFilter(assert_error_log,
line_buffer=ar.syslog_line_buffer) as syslog:
body()
assert stderr.extra_matches == {}
assert syslog.extra_matches == {}
else:
body()
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, False),
'Marathon apps cache has been successfully updated':
SearchCriteria(2, False),
'Cache entry `mesosstate` is too old, aborting request':
SearchCriteria(1, True),
}
ar = nginx_class(cache_poll_period=3,
cache_expiration=2,
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=4,
)
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update using a request:
ping_mesos_agent(ar, superuser_user_header)
# Break mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to be discarded by AR:
# cache_max_age_hard_limit + 1s for good measure
# must be more than cache_poll_period
time.sleep(4 + 1)
# Perform the main/test request:
ping_mesos_agent(ar, superuser_user_header, expect_status=503)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_temp_mesos_borkage_does_not_dirupt_caching(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Mesos state request failed: invalid response status: 500':
SearchCriteria(1, False),
'Marathon apps cache has been successfully updated':
SearchCriteria(2, False),
'Using stale `mesosstate` cache entry to fulfill the request':
SearchCriteria(1, True),
}
ar = nginx_class(cache_poll_period=3,
cache_expiration=2,
cache_max_age_soft_limit=3,
cache_max_age_hard_limit=1800,
)
with GuardedSubprocess(ar):
# Register Line buffer filter:
lbf = LineBufferFilter(filter_regexp,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
# Trigger cache update using a request:
ping_mesos_agent(ar, superuser_user_header)
# Break mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='always_bork',
aux_data=True)
# Wait for the cache to be old enough to become stale:
# cache_max_age_soft_limit + 1s for good measure
time.sleep(3 + 1)
# Perform the main/test request:
ping_mesos_agent(ar, superuser_user_header, expect_status=200)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_first_cache_refresh_occurs_earlier(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Executing cache refresh triggered by timer': SearchCriteria(1, False),
'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True),
'Mesos state cache has been successfully updated': SearchCriteria(1, True),
'Marathon apps cache has been successfully updated': SearchCriteria(1, True),
'Marathon leader cache has been successfully updated': SearchCriteria(1, True),
}
# Enable recording for marathon
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
# Enable recording for Mesos
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
# Make regular polling occur later than usual, so that we get clear
# results.
ar = nginx_class(cache_poll_period=60, cache_expiration=55)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
lbf.scan_log_buffer()
# Do a request that uses cache so that we can verify that data was
# in fact cached and no more than one req to mesos/marathon
# backends were made
ping_mesos_agent(ar, superuser_user_header)
mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert lbf.extra_matches == {}
assert len(mesos_requests) == 1
assert len(marathon_requests) == 2
def test_if_default_scheme_is_honourded_by_mleader_endpoint(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}
cache_poll_period = 3
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=cache_poll_period - 1,
default_scheme="https://")
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 502
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='change_leader',
aux_data="127.0.0.4:443")
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'https://127.0.0.4:443'
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_default_scheme_is_honourded_by_mleader_endpoint(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}
cache_poll_period = 3
ar = nginx_class(cache_poll_period=cache_poll_period,
cache_expiration=cache_poll_period - 1,
default_scheme="https://")
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 502
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='change_leader',
aux_data="127.0.0.4:443")
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
req_data = resp.json()
assert req_data['endpoint_id'] == 'https://127.0.0.4:443'
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_broken_response_from_marathon_is_handled(
self, nginx_class, mocker, superuser_user_header):
filter_regexp = {
'Cannot decode Marathon leader JSON': SearchCriteria(1, True),
}
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='break_leader_reply')
ar = nginx_class()
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
assert resp.status_code == 503
assert lbf.extra_matches == {}
def test_if_broken_response_from_marathon_is_handled(
self, nginx_class, mocker, valid_user_header):
filter_regexp = {
'Cannot decode Marathon leader JSON': SearchCriteria(1, True),
}
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='break_leader_reply')
ar = nginx_class()
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
timeout=(CACHE_FIRST_POLL_DELAY + 1),
line_buffer=ar.stderr_line_buffer)
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
assert resp.status_code == 503
assert lbf.extra_matches == {}
def test_if_mesos_leader_locality_is_resolved(
self, nginx_class, valid_user_header, dns_server_mock):
cache_poll_period = 4
nonlocal_leader_ip = "127.0.0.3"
local_leader_ip = "127.0.0.2"
filter_regexp_pre = {
'Failed to instantiate the resolver': SearchCriteria(0, True),
'mesos leader is non-local: `{}`'.format(nonlocal_leader_ip):
SearchCriteria(1, True),
'Private IP address of the host is unknown, ' +
'aborting cache-entry creation for mesos leader':
SearchCriteria(0, True),
'mesos leader cache has been successfully updated':
SearchCriteria(1, True),
}
filter_regexp_post = {
'Failed to instantiate the resolver': SearchCriteria(0, True),
'mesos leader is local': SearchCriteria(1, True),
'Private IP address of the host is unknown, ' +
'aborting cache-entry creation for mesos leader':
SearchCriteria(0, True),
'mesos leader cache has been successfully updated':
SearchCriteria(1, True),
}
dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip)
ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3)
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp_pre,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip)
# First poll (2s) + normal poll interval(4s) < 2 * normal poll
# interval(4s)
time.sleep(cache_poll_period * 2)
lbf = LineBufferFilter(filter_regexp_post,
line_buffer=ar.stderr_line_buffer)
# Just trigger the cache update:
ping_mesos_agent(ar, valid_user_header)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
def test_if_invalid_auth_attempt_is_logged_correctly(
self, master_ar_process, valid_jwt_generator):
# Create some random, unique user that we can grep for:
uid = 'some_random_string_abc1251231143'
filter_regexp = 'validate_jwt_or_exit\(\): User not found: `{}`'.format(
uid)
lbf = LineBufferFilter(
filter_regexp, line_buffer=master_ar_process.stderr_line_buffer)
# Create token for this user:
token = valid_jwt_generator(uid)
header = {'Authorization': 'token={}'.format(token)}
url = master_ar_process.make_url_from_path()
with lbf:
resp = requests.get(url, allow_redirects=False, headers=header)
assert resp.status_code == 401
assert lbf.all_found
def assert_endpoint_response(
ar,
path,
code,
assert_stderr=None,
headers=None,
cookies=None,
assertions=None
):
"""Asserts response code and log messages in Admin Router stderr for
request against specified path.
Arguments:
ar (Nginx): Running instance of the AR
code (int): Expected response code
assert_stderr (dict): LineBufferFilter compatible definition of messages
to assert
cookies (dict): Optionally provide request cookies
headers (dict): Optionally provide request headers
assertions (List[lambda r]) Optionally provide additional assertions
for the response
"""
def body():
r = requests.get(
ar.make_url_from_path(path),
headers=headers,
cookies=cookies,
)
assert r.status_code == code
if assertions:
for func in assertions:
assert func(r)
if assert_stderr is not None:
lbf = LineBufferFilter(assert_stderr, line_buffer=ar.stderr_line_buffer)
with lbf:
body()
assert lbf.extra_matches == {}
else:
body()
def test_if_mesos_upstream_env_is_honoured(
self, nginx_class, mocker, valid_user_header):
# Stage 0 - setup the environment:
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='record_requests')
# Stage 1 - we set Mesos upstream to http://127.0.0.2:5050 and
# verify that all the requests from cache go there:
filter_regexp = {
'Mesos upstream: http://127.0.0.2:5050': SearchCriteria(1, True),
'Request url: http://127.0.0.2:5050/master/state-summary': SearchCriteria(1, True),
}
ar = nginx_class(upstream_mesos="http://127.0.0.2:5050")
agent_id = AGENT1_ID
url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
assert len(m1_requests) == 1
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='get_recorded_requests')
assert len(m2_requests) == 0
assert lbf.extra_matches == {}
# Stage 1 ends
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='erase_recorded_requests')
# Stage 2 - we set Mesos upstream to http://127.0.0.2:8080 and
# verify that all the requests go to the new upstream
filter_regexp = {
'Mesos upstream: http://127.0.0.3:5050': SearchCriteria(1, True),
'Request url: http://127.0.0.3:5050/master/state-summary': SearchCriteria(1, True),
}
ar = nginx_class(upstream_mesos="http://127.0.0.3:5050")
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
assert len(m1_requests) == 0
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='get_recorded_requests')
assert len(m2_requests) == 1
assert lbf.extra_matches == {}
def test_if_temp_dns_borkage_does_not_disrupt_mesosleader_caching(
self, nginx_class, dns_server_mock, valid_user_header):
filter_regexp_pre = {
'Marathon leader cache has been successfully updated':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(1, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
'`Mesos Leader` state cache has been successfully updated':
SearchCriteria(1, True),
}
filter_regexp_post = {
'Marathon leader cache has been successfully updated':
SearchCriteria(1, True),
'Marathon apps cache has been successfully updated':
SearchCriteria(1, True),
'Mesos state cache has been successfully updated':
SearchCriteria(1, True),
# The problem here is that there may occur two updated, one after
# another, and failed one will be retried. This stems directly from
# how cache.lua works. Let's permit multiple occurences for now.
'DNS server returned error code':
SearchCriteria(1, False),
'Cache entry `mesos_leader` is stale':
SearchCriteria(1, True),
}
cache_max_age_soft_limit = 3
ar = nginx_class(
cache_max_age_soft_limit=cache_max_age_soft_limit,
cache_max_age_hard_limit=1200,
cache_expiration=2,
cache_poll_period=3,
cache_first_poll_delay=1,
)
url = ar.make_url_from_path('/dcos-history-service/foo/bar')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(
filter_regexp_pre,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
with lbf:
# Trigger cache update by issuing request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
assert lbf.extra_matches == {}
lbf = LineBufferFilter(
filter_regexp_post,
timeout=5, # Just to give LBF enough time
line_buffer=ar.stderr_line_buffer)
with lbf:
# Break `leader.mesos` DNS entry
dns_server_mock.remove_dns_entry('leader.mesos.')
# Wait for the cache to be old enough to be considered stale by
# AR:
# cache_max_age_soft_limit + extra delay in order to avoid
# race conditions
delay = 2
time.sleep(cache_max_age_soft_limit + delay)
# Perform the main/test request:
resp = requests.get(url,
allow_redirects=False,
headers=valid_user_header)
assert resp.status_code == 200
assert lbf.extra_matches == {}
def test_if_request_buffering_can_be_configured(
self,
mocker,
nginx_class,
valid_user_header,
label_val,
should_buffer):
# If `DCOS_SERVICE_REQUEST_BUFFERING` is set to `false` (string) or
# `false` (boolean), Admin Router will not buffer the client request before
# sending it to the upstream. In any other case it the request is going
# to be buffered.
# Remove the data from MesosDNS and Mesos mocks w.r.t. resolved service
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=EMPTY_SRV)
# Set the DCOS_SERVICE_REQUEST_BUFFERING for the test mock:
srv = SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT
if label_val is not None:
srv['labels']['DCOS_SERVICE_REQUEST_BUFFERING'] = label_val
new_apps = {"apps": [srv, ]}
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data=new_apps)
# In theory it is possible to write a test that really checks if the
# request was buffered or not. It would require talking to the mocked
# endpoint during the test and checking if it is receiving the data as
# it is being sent (there is no buffering) or only after the whole
# request has been uploaded (Nginx buffers the data). Such a feature
# would introduce some extra complexity into the test harness. Simply
# checking if AR is printing the warning to the error log seems to be
# good enough.
filter_regexp = {}
tmp = 'a client request body is buffered to a temporary file'
if label_val in ["false", False]:
filter_regexp[tmp] = SearchCriteria(0, True)
else:
filter_regexp[tmp] = SearchCriteria(1, True)
ar = nginx_class(role="master")
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
# In order to make Nginx print a warning to the errorlog, the request
# payload needs to be greater than client_body_buffer_size, which by
# default is set to 16k. We use here 2MB for safe measure.
# http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size
payload = {"data": "x" * 1024 * 1024 * 2}
with GuardedSubprocess(ar):
lbf = LineBufferFilter(
filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.post(
url,
allow_redirects=False,
headers=valid_user_header,
data=payload)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
assert resp.status_code == 200
def test_if_marathon_upstream_env_is_honoured(
self, nginx_class, mocker, valid_user_header):
# Stage 0 - setup the environment:
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='record_requests')
# Stage 1 - we set Marathon upstream to http://127.0.0.1:8080 and
# verify that all the requests from cache go there:
filter_regexp = {
'Marathon upstream: http://127.0.0.1:8080': SearchCriteria(1, True),
'Request url: http://127.0.0.1:8080/v2/leader': SearchCriteria(1, True),
('Request url: http://127.0.0.1:8080/v2/apps'
'\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True),
}
ar = nginx_class(upstream_marathon="http://127.0.0.1:8080")
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert len(m1_requests) == 2
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='get_recorded_requests')
assert len(m2_requests) == 0
assert lbf.extra_matches == {}
# Stage 1 ends
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='erase_recorded_requests')
# Stage 2 - we set Marathon upstream to http://127.0.0.2:8080 and
# verify that all the requests go to the new upstream
filter_regexp = {
'Marathon upstream: http://127.0.0.2:8080': SearchCriteria(1, True),
'Request url: http://127.0.0.2:8080/v2/leader': SearchCriteria(1, True),
('Request url: http://127.0.0.2:8080/v2/apps'
'\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True),
}
ar = nginx_class(upstream_marathon="http://127.0.0.2:8080")
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=valid_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert len(m1_requests) == 0
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='get_recorded_requests')
assert len(m2_requests) == 2
assert lbf.extra_matches == {}
def test_if_request_buffering_can_be_configured(self, mocker, nginx_class,
valid_user_header,
label_val, should_buffer):
# If `DCOS_SERVICE_REQUEST_BUFFERING` is set to `false` (string) or
# `false` (boolean), Admin Router will not buffer the client request before
# sending it to the upstream. In any other case it the request is going
# to be buffered.
# Remove the data from MesosDNS and Mesos mocks w.r.t. resolved service
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='set_frameworks_response',
aux_data=[])
mocker.send_command(endpoint_id='http://127.0.0.1:8123',
func_name='set_srv_response',
aux_data=EMPTY_SRV)
# Set the DCOS_SERVICE_REQUEST_BUFFERING for the test mock:
srv = SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT
if label_val is not None:
srv['labels']['DCOS_SERVICE_REQUEST_BUFFERING'] = label_val
new_apps = {
"apps": [
srv,
]
}
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='set_apps_response',
aux_data=new_apps)
# In theory it is possible to write a test that really checks if the
# request was buffered or not. It would require talking to the mocked
# endpoint during the test and checking if it is receiving the data as
# it is being sent (there is no buffering) or only after the whole
# request has been uploaded (Nginx buffers the data). Such a feature
# would introduce some extra complexity into the test harness. Simply
# checking if AR is printing the warning to the error log seems to be
# good enough.
filter_regexp = {}
tmp = 'a client request body is buffered to a temporary file'
if label_val in ["false", False]:
filter_regexp[tmp] = SearchCriteria(0, True)
else:
filter_regexp[tmp] = SearchCriteria(1, True)
ar = nginx_class(role="master")
url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
# In order to make Nginx print a warning to the errorlog, the request
# payload needs to be greater than client_body_buffer_size, which by
# default is set to 16k. We use here 2MB for safe measure.
# http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size
payload = {"data": "x" * 1024 * 1024 * 2}
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
resp = requests.post(url,
allow_redirects=False,
headers=valid_user_header,
data=payload)
lbf.scan_log_buffer()
assert lbf.extra_matches == {}
assert resp.status_code == 200
def test_if_marathon_upstream_env_is_honoured(self, nginx_class, mocker,
superuser_user_header):
# Stage 0 - setup the environment:
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='record_requests')
mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='record_requests')
# Stage 1 - we set Marathon upstream to http://127.0.0.1:8080 and
# verify that all the requests from cache go there:
filter_regexp = {
'Marathon upstream: http://127.0.0.1:8080':
SearchCriteria(1, True),
'Request url: http://127.0.0.1:8080/v2/leader':
SearchCriteria(1, True),
('Request url: http://127.0.0.1:8080/v2/apps'
'\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'):
SearchCriteria(1, True),
}
ar = nginx_class(upstream_marathon="http://127.0.0.1:8080")
url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz')
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert len(m1_requests) == 2
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='get_recorded_requests')
assert len(m2_requests) == 0
assert lbf.extra_matches == {}
# Stage 1 ends
mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='erase_recorded_requests')
# Stage 2 - we set Marathon upstream to http://127.0.0.2:8080 and
# verify that all the requests go to the new upstream
filter_regexp = {
'Marathon upstream: http://127.0.0.2:8080':
SearchCriteria(1, True),
'Request url: http://127.0.0.2:8080/v2/leader':
SearchCriteria(1, True),
('Request url: http://127.0.0.2:8080/v2/apps'
'\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'):
SearchCriteria(1, True),
}
ar = nginx_class(upstream_marathon="http://127.0.0.2:8080")
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
func_name='get_recorded_requests')
assert len(m1_requests) == 0
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080',
func_name='get_recorded_requests')
assert len(m2_requests) == 2
assert lbf.extra_matches == {}
def test_if_mesos_upstream_env_is_honoured(self, nginx_class, mocker,
superuser_user_header):
# Stage 0 - setup the environment:
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='record_requests')
mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='record_requests')
# Stage 1 - we set Mesos upstream to http://127.0.0.2:5050 and
# verify that all the requests from cache go there:
filter_regexp = {
'Mesos upstream: http://127.0.0.2:5050':
SearchCriteria(1, True),
'Request url: http://127.0.0.2:5050/master/state-summary':
SearchCriteria(1, True),
}
ar = nginx_class(upstream_mesos="http://127.0.0.2:5050")
agent_id = 'de1baf83-c36c-4d23-9cb0-f89f596cd6ab-S1'
url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
assert len(m1_requests) == 1
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='get_recorded_requests')
assert len(m2_requests) == 0
assert lbf.extra_matches == {}
# Stage 1 ends
mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='erase_recorded_requests')
# Stage 2 - we set Mesos upstream to http://127.0.0.2:8080 and
# verify that all the requests go to the new upstream
filter_regexp = {
'Mesos upstream: http://127.0.0.3:5050':
SearchCriteria(1, True),
'Request url: http://127.0.0.3:5050/master/state-summary':
SearchCriteria(1, True),
}
ar = nginx_class(upstream_mesos="http://127.0.0.3:5050")
with GuardedSubprocess(ar):
lbf = LineBufferFilter(filter_regexp,
line_buffer=ar.stderr_line_buffer)
requests.get(url,
allow_redirects=False,
headers=superuser_user_header)
lbf.scan_log_buffer()
m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
func_name='get_recorded_requests')
assert len(m1_requests) == 0
m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050',
func_name='get_recorded_requests')
assert len(m2_requests) == 1
assert lbf.extra_matches == {}
