def test_if_broken_marathon_does_not_break_mesos_cache( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), } # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, superuser_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_default_scheme_is_honoured_by_agent_endpoint( self, nginx_class, mocker, valid_user_header): filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)} ar = nginx_class(default_scheme="https://") agent_id = AGENT3_ID url_good = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) agent_id = AGENT1_ID url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url_bad, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 502 resp = requests.get(url_good, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'https://127.0.0.1:15401' lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_upstream_wrong_json( self, nginx_class, mocker, superuser_user_header): filter_regexp = { "Cannot decode Marathon apps JSON: ": SearchCriteria(1, True), } ar = nginx_class() # Set wrong non-json response content mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_encoded_response', aux_data=b"wrong response") url = ar.make_url_from_path('/service/nginx-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert "cache state is invalid" in resp.content.decode('utf-8') assert resp.status_code == 503 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_mesos_does_not_break_marathon_cache( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), } # Break marathon mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='enable_nginx_app') ar = nginx_class() url = ar.make_url_from_path('/service/nginx-enabled/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.1:16001' assert lbf.extra_matches == {}
def test_if_missing_mesos_leader_entry_is_handled( self, nginx_class, valid_user_header, dns_server_mock): filter_regexp = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'DNS server returned error code': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(0, True), } ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Unfortunatelly there are upstreams that use `leader.mesos` and # removing this entry too early will result in Nginx failing to start. # So we need to do it right after nginx starts, but before first # cache update. time.sleep(1) dns_server_mock.remove_dns_entry('leader.mesos.') # Now let's trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, False), 'Mesos state cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `svcapps` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class( cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, cache_expiration=2, cache_poll_period=3, ) mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='enable_nginx_app') url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 503 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_var_pointing_to_empty_file_is_handled(self, ar_process_without_secret_key): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = 'Secret key not set or empty string.' lbf = LineBufferFilter(filter_regexp, line_buffer=ar_process_without_secret_key.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.all_found is True
def test_if_not_defining_the_var_is_handled(self, ar_process_without_secret_key): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = 'SECRET_KEY_FILE_PATH not set.' lbf = LineBufferFilter(filter_regexp, line_buffer=ar_process_without_secret_key.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.all_found is True
def test_if_not_defining_the_var_is_handled(self, nginx_class, role): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = {'SECRET_KEY_FILE_PATH not set.': SearchCriteria(1, False)} ar = nginx_class(role=role, secret_key_file_path=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def _assert_filter_regexp_for_invalid_app( self, filter_regexp, app, nginx_class, mocker, auth_headers, ): """Helper method that will assert if provided regexp filter is found in nginx logs for given apps response from Marathon upstream endpoint. Arguments: filter_regexp (dict): Filter definition where key is the message looked up in logs and value is SearchCriteria definition app (dict): App that upstream endpoint should respond with nginx_class (Nginx): Nginx process fixture mocker (Mocker): Mocker fixture auth_header (dict): Headers that should be passed to Nginx in the request """ ar = nginx_class() mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": [app]}) # Remove all entries for mesos frameworks and mesos_dns so that # we test only the information in Marathon mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=[]) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=auth_headers) assert resp.status_code == 500 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, False), 'Mesos state cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `svcapps` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class(cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, cache_expiration=2, cache_poll_period=3, ) mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='enable_nginx_app') url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 503 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly(self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(3, False), 'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(8, True), 'Mesos state cache has been successfully updated': SearchCriteria(3, True), 'Marathon apps cache has been successfully updated': SearchCriteria(3, True), 'Marathon leader cache has been successfully updated': SearchCriteria(3, True), } cache_poll_period = 4 # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur faster than usual to speed up the tests. ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) # In total, we should get three cache updates in given time frame: timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1 with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=timeout, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 3 assert len(marathon_requests) == 6
def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker): filter_regexp = { 'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True), } ar = nginx_class(host_ip=valid_ip) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_absent_var_is_handled(self, nginx_class, mocker): filter_regexp = { 'Local Mesos Master IP: unknown': SearchCriteria(1, True), } ar = nginx_class(host_ip=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_not_defining_the_var_is_handled(self, ar_process_without_secret_key): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = 'SECRET_KEY_FILE_PATH not set.' lbf = LineBufferFilter( filter_regexp, line_buffer=ar_process_without_secret_key.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.all_found is True
def test_if_var_pointing_to_empty_file_is_handled( self, ar_process_without_secret_key): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = 'Secret key not set or empty string.' lbf = LineBufferFilter( filter_regexp, line_buffer=ar_process_without_secret_key.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.all_found is True
def _assert_filter_regexp_for_invalid_app( self, filter_regexp, app, nginx_class, mocker, auth_headers, ): """Helper method that will assert if provided regexp filter is found in nginx logs for given apps response from Marathon upstream endpoint. Arguments: filter_regexp (dict): Filter definition where key is the message looked up in logs and value is SearchCriteria definition app (dict): App that upstream endpoint should respond with nginx_class (Nginx): Nginx process fixture mocker (Mocker): Mocker fixture auth_header (dict): Headers that should be passed to Nginx in the request """ ar = nginx_class() mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": [app]}) # Remove all entries for mesos frameworks and mesos_dns so that # we test only the information in Marathon mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=[]) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=auth_headers) assert resp.status_code == 404 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_temp_marathon_borkage_does_not_disrupt_caching( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, False), 'Mesos state cache has been successfully updated': SearchCriteria(2, False), 'Using stale `svcapps` cache entry to fulfill the request': SearchCriteria(1, True), } ar = nginx_class( cache_max_age_soft_limit=3, cache_max_age_hard_limit=1200, cache_expiration=2, cache_poll_period=3, ) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be considered stale by AR: # cache_max_age_soft_limit + 1s for a good measure time.sleep(3 + 1) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_is_triggered_by_request(self, nginx_class, mocker, valid_user_header): """...right after Nginx has started.""" filter_regexp = { 'Executing cache refresh triggered by request': SearchCriteria(1, True), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make sure that timers will not interfere: ar = nginx_class(cache_first_poll_delay=120, cache_poll_period=120, cache_expiration=115) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=5, line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() # Do an extra request so that we can verify that data was in fact # cached and no more than one req to mesos/marathon backends were # made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_var_pointing_to_empty_file_is_handled( self, nginx_class, role, empty_file): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = {'Auth token verification key not set': SearchCriteria(1, False)} ar = nginx_class(role=role, auth_token_verification_key_file_path=empty_file) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_temp_marathon_borkage_does_not_disrupt_caching( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, False), 'Mesos state cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `svcapps` is stale': SearchCriteria(1, True), } ar = nginx_class(cache_max_age_soft_limit=3, cache_max_age_hard_limit=1200, cache_expiration=2, cache_poll_period=3, ) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be considered stale by AR: # cache_max_age_soft_limit + 1s for a good measure time.sleep(3 + 1) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_not_defining_the_var_is_handled(self, nginx_class, role): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = { 'AUTH_TOKEN_VERIFICATION_KEY_FILE_PATH not set.': SearchCriteria(1, False) } ar = nginx_class(role=role, auth_token_verification_key_file_path=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_var_is_verified(self, invalid_ip, nginx_class, mocker): filter_regexp = { 'Local Mesos Master IP: unknown': SearchCriteria(1, True), 'HOST_IP var is not a valid ipv4: {}'.format(invalid_ip): SearchCriteria(1, True), } ar = nginx_class(host_ip=invalid_ip) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_public_ip_detect_script_failue_is_handled( self, master_ar_process_perclass, valid_user_header): url = master_ar_process_perclass.make_url_from_path('/metadata') filter_regexp = { 'Traceback \(most recent call last\):': SearchCriteria(1, True), ("FileNotFoundError: \[Errno 2\] No such file or directory:" " '/usr/local/detect_ip_public_data.txt'"): SearchCriteria(1, True), } lbf = LineBufferFilter( filter_regexp, line_buffer=master_ar_process_perclass.stderr_line_buffer) with lbf, overridden_file_content( '/usr/local/detect_ip_public_data.txt'): os.unlink('/usr/local/detect_ip_public_data.txt') resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 assert lbf.extra_matches == {} resp_data = resp.json() assert resp_data['PUBLIC_IPV4'] == "127.0.0.1"
def test_if_valid_auth_attempt_is_logged_correctly( self, master_ar_process, valid_jwt_generator, mocker): # Create some random, unique user that we can grep for: uid = 'some_random_string_abc213421341' mocker.send_command(endpoint_id='http://127.0.0.1:8101', func_name='add_user', aux_data={'uid': uid}) filter_regexp = { 'validate_jwt_or_exit\(\): UID from valid JWT: `{}`'.format(uid): SearchCriteria(1, False)} lbf = LineBufferFilter(filter_regexp, line_buffer=master_ar_process.stderr_line_buffer) # Create token for this user: token = valid_jwt_generator(uid) header = {'Authorization': 'token={}'.format(token)} url = master_ar_process.make_url_from_path() with lbf: resp = requests.get(url, allow_redirects=False, headers=header) assert resp.status_code == 200 assert lbf.extra_matches == {}
def test_if_first_cache_refresh_occurs_earlier(self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(1, False), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for Mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur later than usual, so that we get clear # results. ar = nginx_class(cache_poll_period=60, cache_expiration=55) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_mesos_leader_locality_is_resolved(self, nginx_class, valid_user_header, dns_server_mock): cache_poll_period = 4 nonlocal_leader_ip = "127.0.0.3" local_leader_ip = "127.0.0.2" filter_regexp_pre = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is non-local: `{}`'.format(nonlocal_leader_ip): SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is local': SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip) ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp_pre, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {} dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) lbf = LineBufferFilter(filter_regexp_post, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header): filter_regexp = { 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } ar = nginx_class(host_ip=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_auth_module_is_enabled_by_unless_false_str_is_provided( self, nginx_class, mocker, enable_keyword): filter_regexp = { 'Activate authentication module.': SearchCriteria(1, True), } ar = nginx_class(auth_enabled=enable_keyword) url = ar.make_url_from_path('/exhibitor/foo/bar') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False) assert resp.status_code == 401 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `mesosstate` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class( cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=503) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(3, False), 'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(6, True), 'Mesos state cache has been successfully updated': SearchCriteria(3, True), 'Marathon apps cache has been successfully updated': SearchCriteria(3, True), 'Marathon leader cache has been successfully updated': SearchCriteria(3, True), } cache_poll_period = 4 # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur faster than usual to speed up the tests. ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) # In total, we should get three cache updates in given time frame: timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1 with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=timeout, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, superuser_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 3 assert len(marathon_requests) == 6
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header): filter_regexp = { 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(1, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } ar = nginx_class(host_ip=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_auth_module_can_be_disabled(self, nginx_class, mocker): filter_regexp = { ("ADMINROUTER_ACTIVATE_AUTH_MODULE set to `false`. " "Deactivate authentication module."): SearchCriteria(1, True), } ar = nginx_class(auth_enabled='false') url = ar.make_url_from_path('/exhibitor/foo/bar') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False) assert resp.status_code == 200 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_temp_mesos_borkage_does_not_dirupt_caching( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Using stale `mesosstate` cache entry to fulfill the request': SearchCriteria(1, True), } ar = nginx_class( cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=1800, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to become stale: # cache_max_age_soft_limit + 1s for good measure time.sleep(3 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=200) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_is_triggered_by_request( self, nginx_class, mocker, superuser_user_header): """...right after Nginx has started.""" filter_regexp = { 'Executing cache refresh triggered by request': SearchCriteria(1, True), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make sure that timers will not interfere: ar = nginx_class(cache_first_poll_delay=120, cache_poll_period=120, cache_expiration=115) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=5, line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, superuser_user_header) lbf.scan_log_buffer() # Do an extra request so that we can verify that data was in fact # cached and no more than one req to mesos/marathon backends were # made ping_mesos_agent(ar, superuser_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def assert_endpoint_response(ar, path, code, assert_error_log=None, headers=None, cookies=None, assertions=None): """Asserts response code and log messages in Admin Router stderr for request against specified path. Arguments: ar (Nginx): Running instance of the AR code (int): Expected response code assert_error_log (dict): LineBufferFilter compatible definition of messages to assert cookies (dict): Optionally provide request cookies headers (dict): Optionally provide request headers assertions (List[lambda r]) Optionally provide additional assertions for the response """ def body(): r = requests.get( ar.make_url_from_path(path), headers=headers, cookies=cookies, ) assert r.status_code == code if assertions: for func in assertions: assert func(r) if assert_error_log is not None: # for testing, log messages go to both stderr and to /dev/log with LineBufferFilter(copy.deepcopy(assert_error_log), line_buffer=ar.stderr_line_buffer) as stderr: with LineBufferFilter(assert_error_log, line_buffer=ar.syslog_line_buffer) as syslog: body() assert stderr.extra_matches == {} assert syslog.extra_matches == {} else: body()
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `mesosstate` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class(cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, superuser_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: ping_mesos_agent(ar, superuser_user_header, expect_status=503) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_temp_mesos_borkage_does_not_dirupt_caching( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Using stale `mesosstate` cache entry to fulfill the request': SearchCriteria(1, True), } ar = nginx_class(cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=1800, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, superuser_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to become stale: # cache_max_age_soft_limit + 1s for good measure time.sleep(3 + 1) # Perform the main/test request: ping_mesos_agent(ar, superuser_user_header, expect_status=200) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_first_cache_refresh_occurs_earlier( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(1, False), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for Mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur later than usual, so that we get clear # results. ar = nginx_class(cache_poll_period=60, cache_expiration=55) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, superuser_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_default_scheme_is_honourded_by_mleader_endpoint( self, nginx_class, mocker, superuser_user_header): filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)} cache_poll_period = 3 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=cache_poll_period - 1, default_scheme="https://") url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 502 mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='change_leader', aux_data="127.0.0.4:443") # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'https://127.0.0.4:443' lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_default_scheme_is_honourded_by_mleader_endpoint( self, nginx_class, mocker, valid_user_header): filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)} cache_poll_period = 3 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=cache_poll_period - 1, default_scheme="https://") url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 502 mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='change_leader', aux_data="127.0.0.4:443") # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'https://127.0.0.4:443' lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_response_from_marathon_is_handled( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Cannot decode Marathon leader JSON': SearchCriteria(1, True), } mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='break_leader_reply') ar = nginx_class() url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() assert resp.status_code == 503 assert lbf.extra_matches == {}
def test_if_broken_response_from_marathon_is_handled( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Cannot decode Marathon leader JSON': SearchCriteria(1, True), } mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='break_leader_reply') ar = nginx_class() url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() assert resp.status_code == 503 assert lbf.extra_matches == {}
def test_if_mesos_leader_locality_is_resolved( self, nginx_class, valid_user_header, dns_server_mock): cache_poll_period = 4 nonlocal_leader_ip = "127.0.0.3" local_leader_ip = "127.0.0.2" filter_regexp_pre = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'mesos leader is non-local: `{}`'.format(nonlocal_leader_ip): SearchCriteria(1, True), 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(0, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'mesos leader is local': SearchCriteria(1, True), 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(0, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip) ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp_pre, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {} dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) lbf = LineBufferFilter(filter_regexp_post, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_invalid_auth_attempt_is_logged_correctly( self, master_ar_process, valid_jwt_generator): # Create some random, unique user that we can grep for: uid = 'some_random_string_abc1251231143' filter_regexp = 'validate_jwt_or_exit\(\): User not found: `{}`'.format( uid) lbf = LineBufferFilter( filter_regexp, line_buffer=master_ar_process.stderr_line_buffer) # Create token for this user: token = valid_jwt_generator(uid) header = {'Authorization': 'token={}'.format(token)} url = master_ar_process.make_url_from_path() with lbf: resp = requests.get(url, allow_redirects=False, headers=header) assert resp.status_code == 401 assert lbf.all_found
def assert_endpoint_response( ar, path, code, assert_stderr=None, headers=None, cookies=None, assertions=None ): """Asserts response code and log messages in Admin Router stderr for request against specified path. Arguments: ar (Nginx): Running instance of the AR code (int): Expected response code assert_stderr (dict): LineBufferFilter compatible definition of messages to assert cookies (dict): Optionally provide request cookies headers (dict): Optionally provide request headers assertions (List[lambda r]) Optionally provide additional assertions for the response """ def body(): r = requests.get( ar.make_url_from_path(path), headers=headers, cookies=cookies, ) assert r.status_code == code if assertions: for func in assertions: assert func(r) if assert_stderr is not None: lbf = LineBufferFilter(assert_stderr, line_buffer=ar.stderr_line_buffer) with lbf: body() assert lbf.extra_matches == {} else: body()
def test_if_mesos_upstream_env_is_honoured( self, nginx_class, mocker, valid_user_header): # Stage 0 - setup the environment: mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='record_requests') # Stage 1 - we set Mesos upstream to http://127.0.0.2:5050 and # verify that all the requests from cache go there: filter_regexp = { 'Mesos upstream: http://127.0.0.2:5050': SearchCriteria(1, True), 'Request url: http://127.0.0.2:5050/master/state-summary': SearchCriteria(1, True), } ar = nginx_class(upstream_mesos="http://127.0.0.2:5050") agent_id = AGENT1_ID url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') assert len(m1_requests) == 1 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='get_recorded_requests') assert len(m2_requests) == 0 assert lbf.extra_matches == {} # Stage 1 ends mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='erase_recorded_requests') # Stage 2 - we set Mesos upstream to http://127.0.0.2:8080 and # verify that all the requests go to the new upstream filter_regexp = { 'Mesos upstream: http://127.0.0.3:5050': SearchCriteria(1, True), 'Request url: http://127.0.0.3:5050/master/state-summary': SearchCriteria(1, True), } ar = nginx_class(upstream_mesos="http://127.0.0.3:5050") with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') assert len(m1_requests) == 0 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='get_recorded_requests') assert len(m2_requests) == 1 assert lbf.extra_matches == {}
def test_if_temp_dns_borkage_does_not_disrupt_mesosleader_caching( self, nginx_class, dns_server_mock, valid_user_header): filter_regexp_pre = { 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), # The problem here is that there may occur two updated, one after # another, and failed one will be retried. This stems directly from # how cache.lua works. Let's permit multiple occurences for now. 'DNS server returned error code': SearchCriteria(1, False), 'Cache entry `mesos_leader` is stale': SearchCriteria(1, True), } cache_max_age_soft_limit = 3 ar = nginx_class( cache_max_age_soft_limit=cache_max_age_soft_limit, cache_max_age_hard_limit=1200, cache_expiration=2, cache_poll_period=3, cache_first_poll_delay=1, ) url = ar.make_url_from_path('/dcos-history-service/foo/bar') with GuardedSubprocess(ar): lbf = LineBufferFilter( filter_regexp_pre, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) with lbf: # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 assert lbf.extra_matches == {} lbf = LineBufferFilter( filter_regexp_post, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) with lbf: # Break `leader.mesos` DNS entry dns_server_mock.remove_dns_entry('leader.mesos.') # Wait for the cache to be old enough to be considered stale by # AR: # cache_max_age_soft_limit + extra delay in order to avoid # race conditions delay = 2 time.sleep(cache_max_age_soft_limit + delay) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 assert lbf.extra_matches == {}
def test_if_request_buffering_can_be_configured( self, mocker, nginx_class, valid_user_header, label_val, should_buffer): # If `DCOS_SERVICE_REQUEST_BUFFERING` is set to `false` (string) or # `false` (boolean), Admin Router will not buffer the client request before # sending it to the upstream. In any other case it the request is going # to be buffered. # Remove the data from MesosDNS and Mesos mocks w.r.t. resolved service mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=EMPTY_SRV) # Set the DCOS_SERVICE_REQUEST_BUFFERING for the test mock: srv = SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT if label_val is not None: srv['labels']['DCOS_SERVICE_REQUEST_BUFFERING'] = label_val new_apps = {"apps": [srv, ]} mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data=new_apps) # In theory it is possible to write a test that really checks if the # request was buffered or not. It would require talking to the mocked # endpoint during the test and checking if it is receiving the data as # it is being sent (there is no buffering) or only after the whole # request has been uploaded (Nginx buffers the data). Such a feature # would introduce some extra complexity into the test harness. Simply # checking if AR is printing the warning to the error log seems to be # good enough. filter_regexp = {} tmp = 'a client request body is buffered to a temporary file' if label_val in ["false", False]: filter_regexp[tmp] = SearchCriteria(0, True) else: filter_regexp[tmp] = SearchCriteria(1, True) ar = nginx_class(role="master") url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') # In order to make Nginx print a warning to the errorlog, the request # payload needs to be greater than client_body_buffer_size, which by # default is set to 16k. We use here 2MB for safe measure. # http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size payload = {"data": "x" * 1024 * 1024 * 2} with GuardedSubprocess(ar): lbf = LineBufferFilter( filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.post( url, allow_redirects=False, headers=valid_user_header, data=payload) lbf.scan_log_buffer() assert lbf.extra_matches == {} assert resp.status_code == 200
def test_if_marathon_upstream_env_is_honoured( self, nginx_class, mocker, valid_user_header): # Stage 0 - setup the environment: mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='record_requests') # Stage 1 - we set Marathon upstream to http://127.0.0.1:8080 and # verify that all the requests from cache go there: filter_regexp = { 'Marathon upstream: http://127.0.0.1:8080': SearchCriteria(1, True), 'Request url: http://127.0.0.1:8080/v2/leader': SearchCriteria(1, True), ('Request url: http://127.0.0.1:8080/v2/apps' '\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True), } ar = nginx_class(upstream_marathon="http://127.0.0.1:8080") url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert len(m1_requests) == 2 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='get_recorded_requests') assert len(m2_requests) == 0 assert lbf.extra_matches == {} # Stage 1 ends mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='erase_recorded_requests') # Stage 2 - we set Marathon upstream to http://127.0.0.2:8080 and # verify that all the requests go to the new upstream filter_regexp = { 'Marathon upstream: http://127.0.0.2:8080': SearchCriteria(1, True), 'Request url: http://127.0.0.2:8080/v2/leader': SearchCriteria(1, True), ('Request url: http://127.0.0.2:8080/v2/apps' '\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True), } ar = nginx_class(upstream_marathon="http://127.0.0.2:8080") with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert len(m1_requests) == 0 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='get_recorded_requests') assert len(m2_requests) == 2 assert lbf.extra_matches == {}
def test_if_request_buffering_can_be_configured(self, mocker, nginx_class, valid_user_header, label_val, should_buffer): # If `DCOS_SERVICE_REQUEST_BUFFERING` is set to `false` (string) or # `false` (boolean), Admin Router will not buffer the client request before # sending it to the upstream. In any other case it the request is going # to be buffered. # Remove the data from MesosDNS and Mesos mocks w.r.t. resolved service mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=EMPTY_SRV) # Set the DCOS_SERVICE_REQUEST_BUFFERING for the test mock: srv = SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT if label_val is not None: srv['labels']['DCOS_SERVICE_REQUEST_BUFFERING'] = label_val new_apps = { "apps": [ srv, ] } mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data=new_apps) # In theory it is possible to write a test that really checks if the # request was buffered or not. It would require talking to the mocked # endpoint during the test and checking if it is receiving the data as # it is being sent (there is no buffering) or only after the whole # request has been uploaded (Nginx buffers the data). Such a feature # would introduce some extra complexity into the test harness. Simply # checking if AR is printing the warning to the error log seems to be # good enough. filter_regexp = {} tmp = 'a client request body is buffered to a temporary file' if label_val in ["false", False]: filter_regexp[tmp] = SearchCriteria(0, True) else: filter_regexp[tmp] = SearchCriteria(1, True) ar = nginx_class(role="master") url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') # In order to make Nginx print a warning to the errorlog, the request # payload needs to be greater than client_body_buffer_size, which by # default is set to 16k. We use here 2MB for safe measure. # http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size payload = {"data": "x" * 1024 * 1024 * 2} with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.post(url, allow_redirects=False, headers=valid_user_header, data=payload) lbf.scan_log_buffer() assert lbf.extra_matches == {} assert resp.status_code == 200
def test_if_marathon_upstream_env_is_honoured(self, nginx_class, mocker, superuser_user_header): # Stage 0 - setup the environment: mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='record_requests') # Stage 1 - we set Marathon upstream to http://127.0.0.1:8080 and # verify that all the requests from cache go there: filter_regexp = { 'Marathon upstream: http://127.0.0.1:8080': SearchCriteria(1, True), 'Request url: http://127.0.0.1:8080/v2/leader': SearchCriteria(1, True), ('Request url: http://127.0.0.1:8080/v2/apps' '\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True), } ar = nginx_class(upstream_marathon="http://127.0.0.1:8080") url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert len(m1_requests) == 2 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='get_recorded_requests') assert len(m2_requests) == 0 assert lbf.extra_matches == {} # Stage 1 ends mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='erase_recorded_requests') # Stage 2 - we set Marathon upstream to http://127.0.0.2:8080 and # verify that all the requests go to the new upstream filter_regexp = { 'Marathon upstream: http://127.0.0.2:8080': SearchCriteria(1, True), 'Request url: http://127.0.0.2:8080/v2/leader': SearchCriteria(1, True), ('Request url: http://127.0.0.2:8080/v2/apps' '\?embed=apps\.tasks\&label=DCOS_SERVICE_NAME'): SearchCriteria(1, True), } ar = nginx_class(upstream_marathon="http://127.0.0.2:8080") with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert len(m1_requests) == 0 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.2:8080', func_name='get_recorded_requests') assert len(m2_requests) == 2 assert lbf.extra_matches == {}
def test_if_mesos_upstream_env_is_honoured(self, nginx_class, mocker, superuser_user_header): # Stage 0 - setup the environment: mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='record_requests') # Stage 1 - we set Mesos upstream to http://127.0.0.2:5050 and # verify that all the requests from cache go there: filter_regexp = { 'Mesos upstream: http://127.0.0.2:5050': SearchCriteria(1, True), 'Request url: http://127.0.0.2:5050/master/state-summary': SearchCriteria(1, True), } ar = nginx_class(upstream_mesos="http://127.0.0.2:5050") agent_id = 'de1baf83-c36c-4d23-9cb0-f89f596cd6ab-S1' url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') assert len(m1_requests) == 1 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='get_recorded_requests') assert len(m2_requests) == 0 assert lbf.extra_matches == {} # Stage 1 ends mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='erase_recorded_requests') # Stage 2 - we set Mesos upstream to http://127.0.0.2:8080 and # verify that all the requests go to the new upstream filter_regexp = { 'Mesos upstream: http://127.0.0.3:5050': SearchCriteria(1, True), 'Request url: http://127.0.0.3:5050/master/state-summary': SearchCriteria(1, True), } ar = nginx_class(upstream_mesos="http://127.0.0.3:5050") with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) requests.get(url, allow_redirects=False, headers=superuser_user_header) lbf.scan_log_buffer() m1_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') assert len(m1_requests) == 0 m2_requests = mocker.send_command(endpoint_id='http://127.0.0.3:5050', func_name='get_recorded_requests') assert len(m2_requests) == 1 assert lbf.extra_matches == {}