def test_if_early_boot_stage_can_recover_from_a_bit_slow_backend( self, nginx_class, valid_user_header, mocker, log_catcher): # The idea here is to make Backend a bit slow, so that AR is still able # to update cache on first request. refresh_lock_timeout = 10 backend_request_timeout = 5 ar = nginx_class( cache_first_poll_delay=1, cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, cache_backend_request_timeout=backend_request_timeout, cache_refresh_lock_timeout=refresh_lock_timeout, ) agent_id = AGENT1_ID url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) v = Vegeta(log_catcher, target=url, jwt=valid_user_header, rate=3) # Make mesos just a bit :) # It mus respond slower than backend_request_timeout mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_stall', aux_data=backend_request_timeout * 0.3) with GuardedSubprocess(ar): with GuardedSubprocess(v): time.sleep(backend_request_timeout * 0.3 + 1) # let it warm-up! ping_mesos_agent(ar, valid_user_header)
def test_if_broken_mesos_does_not_prevent_resolving_via_marathon( self, mocker, nginx_class, valid_user_header): # Bork Mesos Mock, Make MesosDNS mock respond with no apps, so that AR # is able to resolve only via Marathon/we are certain that it resolved # via Marathon. mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=EMPTY_SRV) # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class( cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) url = ar.make_url_from_path("/service/scheduler-alwaysthere/") with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 data = resp.json() assert data['endpoint_id'] == 'http://127.0.0.1:16000'
def test_ip_per_task_app_with_unspecified_ip_address_DCOS_OSS_1366( self, nginx_class, mocker, valid_user_header): """ Test that an app that, instead of specifying 'ipAddress: null' does not specify 'ipAddress' at all, is successfully cached. """ app = self._scheduler_alwaysthere_app() # Remove the 'ipAddress' key completely, thereby triggering DCOS_OSS-1366. del (app["ipAddress"]) ar = nginx_class() mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": [app]}) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.1:16000'
def test_ip_per_task_app_without_user_networking(self, nginx_class, mocker, valid_user_header): app = self._scheduler_alwaysthere_app() app['ipAddress'] = { 'networkName': 'samplenet', 'discovery': { "ports": [{ "number": 80, "name": "http", "protocol": "tcp" }] } } app['tasks'][0]['ipAddresses'][0]['ipAddress'] = '127.0.0.2' ar = nginx_class() mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": [app]}) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.2:80'
def test_mesos_leader_reresolve_in_proxy_pass( self, nginx_class, valid_user_header, dns_server_mock, path, dest_port, ): # Change the TTL of `leader.mesos.` entry dns_server_mock.set_dns_entry( 'leader.mesos.', ip='127.0.0.2', ttl=self.LONG_TTL) ar = nginx_class(upstream_mesos="http://leader.mesos:5050") with GuardedSubprocess(ar): generic_correct_upstream_dest_test( ar, valid_user_header, path, "http://127.0.0.2:{}".format(dest_port), ) # Update the `leader.mesos.` entry with new value dns_server_mock.set_dns_entry( 'leader.mesos.', ip='127.0.0.3', ttl=self.LONG_TTL) # This should be equal to 1.5 times the value of `valid=` DNS TTL # override in `resolver` config option -> 5s * 1.5 = 7.5s time.sleep(5 * 1.5) generic_correct_upstream_dest_test( ar, valid_user_header, path, "http://127.0.0.3:{}".format(dest_port), )
def test_if_default_scheme_is_honoured_by_agent_endpoint( self, nginx_class, mocker, valid_user_header): filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)} ar = nginx_class(default_scheme="https://") agent_id = AGENT3_ID url_good = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) agent_id = AGENT1_ID url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url_bad, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 502 resp = requests.get(url_good, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'https://127.0.0.1:15401' lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_default_scheme_is_honoured_by_agent_endpoint( self, nginx_class, mocker, superuser_user_header): filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)} ar = nginx_class(default_scheme="https://") agent_id = '35f210bb-bb58-4559-9932-b62619e72b6d-S0' url_good = ar.make_url_from_path( '/agent/{}/blah/blah'.format(agent_id)) agent_id = 'de1baf83-c36c-4d23-9cb0-f89f596cd6ab-S1' url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) resp = requests.get(url_bad, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 502 resp = requests.get(url_good, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'https://127.0.0.1:15401' lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_broken_mesos_does_not_break_marathon_cache( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), } # Break marathon mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) ar = nginx_class() url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz') with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) resp = requests.get(url, allow_redirects=False, headers=valid_user_header) lbf.scan_log_buffer() assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.1:16000' assert lbf.extra_matches == {}
def test_if_caching_works_for_marathon_apps(self, nginx_class, mocker, valid_user_header): # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') ar = nginx_class() url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz') with GuardedSubprocess(ar): # Let the cache warm-up: time.sleep(CACHE_FIRST_POLL_DELAY + 1) for _ in range(5): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') # 3 requests + only one upstream requst == cache works assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_broken_marathon_does_not_prevent_resolving_root_marathon( self, mocker, nginx_class, valid_user_header): # Bork Marathon Mock, DO NOT touch Mesos Mock: mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Change the Root Marathon's endpoint address as reported by Mesos mock, # so that we do not get the reply from the original endpoint (i.e. # http://127.0.0.1:8080 as it will always respond with broken responses # (see `mocker.send_command` call above). fwrk = framework_from_template(SCHEDULER_FWRK_ALWAYSTHERE_ID, "marathon", "http://127.0.0.2:8080/") mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[fwrk]) # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class( cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) url = ar.make_url_from_path("/service/marathon/v2/reflect/me") with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 data = resp.json() assert data['endpoint_id'] == 'http://127.0.0.2:8080'
def test_if_broken_marathon_does_not_prevent_resolving_root_metronome( self, mocker, nginx_class, valid_user_header): # Bork Marathon Mock, DO NOT touch Mesos Mock: mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class( cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) url = ar.make_url_from_path("/service/metronome/foo/bar") with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 data = resp.json() assert data['endpoint_id'] == 'http://127.0.0.1:9000'
def test_upstream_wrong_json(self, nginx_class, mocker, valid_user_header): filter_regexp = { "Cannot decode Marathon apps JSON: ": SearchCriteria(1, True), } ar = nginx_class() # Set wrong non-json response content mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_encoded_response', aux_data=b"wrong response") url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert "cache state is invalid" in resp.content.decode('utf-8') assert resp.status_code == 503 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_missing_mesos_leader_entry_is_handled(self, nginx_class, valid_user_header, dns_server_mock): filter_regexp = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'DNS server returned error code': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(0, True), } ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Unfortunatelly there are upstreams that use `leader.mesos` and # removing this entry too early will result in Nginx failing to start. # So we need to do it right after nginx starts, but before first # cache update. time.sleep(1) dns_server_mock.remove_dns_entry('leader.mesos.') # Now let's trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_caching_works_for_marathon_leader(self, nginx_class, mocker, valid_user_header): # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') ar = nginx_class() url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): # Let the cache warm-up: time.sleep(CACHE_FIRST_POLL_DELAY + 1) for _ in range(5): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.2:80' marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') # 3 requests + only one upstream request == cache works assert len(marathon_requests) == 2
def test_if_changing_marathon_leader_is_reflected_in_cache( self, nginx_class, mocker, valid_user_header): cache_poll_period = 4 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) url = ar.make_url_from_path('/system/v1/leader/marathon/foo/bar/baz') with GuardedSubprocess(ar): # let's make sure that current leader is the default one resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.2:80' # change the leader and wait for cache to notice mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='change_leader', aux_data="127.0.0.3:80") # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) # now, let's see if the leader changed resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.3:80'
def test_if_broken_mesos_prevents_resolving_via_mesosdns( self, mocker, nginx_class, valid_user_header): # Bork Mesos Mock, Make Marathon mock respond with no apps, so that AR # tries to resolve via Mesos /state-summary mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": []}) mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class( cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) url = ar.make_url_from_path("/service/scheduler-alwaysthere/") with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 503 assert '503 Service Unavailable: invalid Mesos state cache' == resp.text.strip( )
def test_if_changing_marathon_apps_is_reflected_in_cache( self, nginx_class, valid_user_header, mocker): cache_poll_period = 4 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) url = ar.make_url_from_path('/service/scheduler-alwaysthere/bar/baz') with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.1:16000' new_apps = { "apps": [ SCHEDULER_APP_ALWAYSTHERE_DIFFERENTPORT, ] } mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data=new_apps) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.15:16001'
def test_if_broken_marathon_prevents_resolving_via_mesos_state_summary( self, mocker, nginx_class, valid_user_header): # Bork Marathon Mock, DO NOT touch Mesos Mock: mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class( cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) url = ar.make_url_from_path("/service/scheduler-alwaysthere/") with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 503 assert '503 Service Unavailable: invalid Marathon svcapps cache' in resp.text
def test_if_broken_marathon_does_not_break_mesos_cache( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), } # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_changing_marathon_apps_is_reflected_in_cache( self, nginx_class, superuser_user_header, mocker): cache_poll_period = 4 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) url = ar.make_url_from_path('/service/nginx-enabled/bar/baz') with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 500 mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='enable_nginx_app') # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 req_data = resp.json() assert req_data['endpoint_id'] == 'http://127.0.0.1:16001'
def test_if_mesos_leader_locality_is_resolved(self, nginx_class, valid_user_header, dns_server_mock): cache_poll_period = 4 nonlocal_leader_ip = "127.0.0.3" local_leader_ip = "127.0.0.2" filter_regexp_pre = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is non-local: `{}`'.format(nonlocal_leader_ip): SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is local': SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip) ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp_pre, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {} dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) lbf = LineBufferFilter(filter_regexp_post, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_absence_of_agent_is_handled_by_cache(self, nginx_class, mocker, valid_user_header): ar = nginx_class() with GuardedSubprocess(ar): ping_mesos_agent( ar, valid_user_header, agent_id='bdcd424a-b59e-4df4-b492-b54e38926bd8-S0', expect_status=404)
def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, superuser_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, False), 'Mesos state cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `svcapps` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class( cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, cache_expiration=2, cache_poll_period=3, ) mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='enable_nginx_app') url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 200 # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: resp = requests.get(url, allow_redirects=False, headers=superuser_user_header) assert resp.status_code == 503 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_leader_is_unknown_state_is_handled( self, nginx_class, valid_user_header): ar = nginx_class(host_ip=None) url = ar.make_url_from_path('/dcos-history-service/foo/bar') with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 503 assert 'mesos leader is unknown' in resp.text
def test_if_historyservice_endpoint_auth_precedence_is_enforced( self, valid_user_header, mocker, nginx_class): ar = nginx_class(host_ip=None) url = ar.make_url_from_path('/dcos-history-service/foo/bar') with GuardedSubprocess(ar): resp = requests.get(url, allow_redirects=False) assert resp.status_code == 401 resp = requests.get(url, allow_redirects=False, headers=valid_user_header) assert resp.status_code == 503
def test_if_not_defining_the_var_is_handled(self, nginx_class, role): # Scanning for the exact log entry is bad, but in this case - can't be # avoided. filter_regexp = {'SECRET_KEY_FILE_PATH not set.': SearchCriteria(1, False)} ar = nginx_class(role=role, secret_key_file_path=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_ar_with_empty_cache_waits_for_marathon_during_service_resolve( self, mocker, nginx_class, valid_user_header): # Make service endpoint resolve only Marathon-related data: mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command( endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=EMPTY_SRV) # Make Mock endpoint stall a little, make sure AR cache update timeouts # are big enough to swallow it: backend_request_timeout = 6 refresh_lock_timeout = backend_request_timeout * 2 # Make period cache refreshes so rare that they do not get into # picture: ar = nginx_class(cache_first_poll_delay=1200, cache_poll_period=1200, cache_expiration=1200, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, cache_backend_request_timeout=backend_request_timeout, cache_refresh_lock_timeout=refresh_lock_timeout, ) url = ar.make_url_from_path("/service/scheduler-alwaysthere/") mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_stall', aux_data=backend_request_timeout * 0.5) # Measure the time it took and the results: with GuardedSubprocess(ar): t_start = time.time() resp = requests.get(url, allow_redirects=False, headers=valid_user_header) t_spent = time.time() - t_start assert resp.status_code == 200 data = resp.json() assert data['endpoint_id'] == 'http://127.0.0.1:16000' # If AR waits for cache during resolve, then time spent should be # greater than the stall time that has been set. Due to the fact # that update coroutines are not separated yet, this will be # slightly higher than: 2 * (backend_request_timeout * 0.5) # as we have two calls to Marathon (svcapps + marathon leader) from # the cache code. assert t_spent > 2 * (backend_request_timeout * 0.5)
def test_if_cache_refresh_occurs_regularly(self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(3, False), 'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(8, True), 'Mesos state cache has been successfully updated': SearchCriteria(3, True), 'Marathon apps cache has been successfully updated': SearchCriteria(3, True), 'Marathon leader cache has been successfully updated': SearchCriteria(3, True), } cache_poll_period = 4 # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur faster than usual to speed up the tests. ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) # In total, we should get three cache updates in given time frame: timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1 with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=timeout, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 3 assert len(marathon_requests) == 6
def _assert_filter_regexp_for_invalid_app( self, filter_regexp, app, nginx_class, mocker, auth_headers, ): """Helper method that will assert if provided regexp filter is found in nginx logs for given apps response from Marathon upstream endpoint. Arguments: filter_regexp (dict): Filter definition where key is the message looked up in logs and value is SearchCriteria definition app (dict): App that upstream endpoint should respond with nginx_class (Nginx): Nginx process fixture mocker (Mocker): Mocker fixture auth_header (dict): Headers that should be passed to Nginx in the request """ ar = nginx_class() mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='set_apps_response', aux_data={"apps": [app]}) # Remove all entries for mesos frameworks and mesos_dns so that # we test only the information in Marathon mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='set_frameworks_response', aux_data=[]) mocker.send_command(endpoint_id='http://127.0.0.1:8123', func_name='set_srv_response', aux_data=[]) url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/') with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update by issuing request: resp = requests.get(url, allow_redirects=False, headers=auth_headers) assert resp.status_code == 500 lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker): filter_regexp = { 'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True), } ar = nginx_class(host_ip=valid_ip) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() assert lbf.extra_matches == {}