def test_if_broken_marathon_does_not_break_mesos_cache( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Marathon app request failed: invalid response status: 500': SearchCriteria(1, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), } # Break marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='always_bork', aux_data=True) ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_early_boot_stage_can_recover_from_a_bit_slow_backend( self, nginx_class, valid_user_header, mocker, log_catcher): # The idea here is to make Backend a bit slow, so that AR is still able # to update cache on first request. refresh_lock_timeout = 10 backend_request_timeout = 5 ar = nginx_class(cache_first_poll_delay=1, cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, cache_backend_request_timeout=backend_request_timeout, cache_refresh_lock_timeout=refresh_lock_timeout, ) agent_id = AGENT1_ID url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) v = Vegeta(log_catcher, target=url, jwt=valid_user_header, rate=3) # Make mesos just a bit :) # It mus respond slower than backend_request_timeout mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_stall', aux_data=backend_request_timeout * 0.3) with GuardedSubprocess(ar): with GuardedSubprocess(v): time.sleep(backend_request_timeout * 0.3 + 1) # let it warm-up! ping_mesos_agent(ar, valid_user_header)
def test_if_missing_mesos_leader_entry_is_handled( self, nginx_class, valid_user_header, dns_server_mock): filter_regexp = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'DNS server returned error code': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(0, True), } ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Unfortunatelly there are upstreams that use `leader.mesos` and # removing this entry too early will result in Nginx failing to start. # So we need to do it right after nginx starts, but before first # cache update. time.sleep(1) dns_server_mock.remove_dns_entry('leader.mesos.') # Now let's trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_early_boot_stage_can_recover_from_a_bit_slow_backend( self, nginx_class, valid_user_header, mocker, log_catcher): # The idea here is to make Backend a bit slow, so that AR is still able # to update cache on first request. refresh_lock_timeout = 10 backend_request_timeout = 5 ar = nginx_class( cache_first_poll_delay=1, cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, cache_backend_request_timeout=backend_request_timeout, cache_refresh_lock_timeout=refresh_lock_timeout, ) agent_id = AGENT1_ID url = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id)) v = Vegeta(log_catcher, target=url, jwt=valid_user_header, rate=3) # Make mesos just a bit :) # It mus respond slower than backend_request_timeout mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_stall', aux_data=backend_request_timeout * 0.3) with GuardedSubprocess(ar): with GuardedSubprocess(v): time.sleep(backend_request_timeout * 0.3 + 1) # let it warm-up! ping_mesos_agent(ar, valid_user_header)
def test_if_backend_requests_have_useragent_set_correctly( self, nginx_class, mocker, valid_user_header): # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for Mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur later than usual, so that we get a single # cache refresh: ar = nginx_class(cache_poll_period=60, cache_expiration=55) with GuardedSubprocess(ar): # Initiate cache refresh by issuing a request: ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert len(mesos_requests) == 1 assert len(marathon_requests) == 2 # We could use a loop here, but let's make it a bit easier to debug: verify_header(mesos_requests[0]['headers'], 'User-Agent', 'Master Admin Router') verify_header(marathon_requests[0]['headers'], 'User-Agent', 'Master Admin Router') verify_header(marathon_requests[1]['headers'], 'User-Agent', 'Master Admin Router')
def test_if_missing_mesos_leader_entry_is_handled(self, nginx_class, valid_user_header, dns_server_mock): filter_regexp = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'DNS server returned error code': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(0, True), } ar = nginx_class() with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Unfortunatelly there are upstreams that use `leader.mesos` and # removing this entry too early will result in Nginx failing to start. # So we need to do it right after nginx starts, but before first # cache update. time.sleep(1) dns_server_mock.remove_dns_entry('leader.mesos.') # Now let's trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_mesos_leader_locality_is_resolved(self, nginx_class, valid_user_header, dns_server_mock): cache_poll_period = 4 nonlocal_leader_ip = "127.0.0.3" local_leader_ip = "127.0.0.2" filter_regexp_pre = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is non-local: `{}`'.format(nonlocal_leader_ip): SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'Mesos Leader is local': SearchCriteria(1, True), 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(0, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip) ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp_pre, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {} dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) lbf = LineBufferFilter(filter_regexp_post, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_absence_of_agent_is_handled_by_cache(self, nginx_class, mocker, valid_user_header): ar = nginx_class() with GuardedSubprocess(ar): ping_mesos_agent( ar, valid_user_header, agent_id='bdcd424a-b59e-4df4-b492-b54e38926bd8-S0', expect_status=404)
def test_if_absence_of_agent_is_handled_by_cache( self, nginx_class, mocker, valid_user_header): ar = nginx_class() with GuardedSubprocess(ar): ping_mesos_agent( ar, valid_user_header, agent_id='bdcd424a-b59e-4df4-b492-b54e38926bd8-S0', expect_status=404)
def test_if_mesos_leader_locality_is_resolved( self, nginx_class, valid_user_header, dns_server_mock): cache_poll_period = 4 nonlocal_leader_ip = "127.0.0.3" local_leader_ip = "127.0.0.2" filter_regexp_pre = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'mesos leader is non-local: `{}`'.format(nonlocal_leader_ip): SearchCriteria(1, True), 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(0, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } filter_regexp_post = { 'Failed to instantiate the resolver': SearchCriteria(0, True), 'mesos leader is local': SearchCriteria(1, True), 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(0, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } dns_server_mock.set_dns_entry('leader.mesos.', ip=nonlocal_leader_ip) ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp_pre, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {} dns_server_mock.set_dns_entry('leader.mesos.', ip=local_leader_ip) # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) lbf = LineBufferFilter(filter_regexp_post, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly(self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(3, False), 'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(8, True), 'Mesos state cache has been successfully updated': SearchCriteria(3, True), 'Marathon apps cache has been successfully updated': SearchCriteria(3, True), 'Marathon leader cache has been successfully updated': SearchCriteria(3, True), } cache_poll_period = 4 # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur faster than usual to speed up the tests. ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) # In total, we should get three cache updates in given time frame: timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1 with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=timeout, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 3 assert len(marathon_requests) == 6
def test_if_cache_refresh_is_triggered_by_request(self, nginx_class, mocker, valid_user_header): """...right after Nginx has started.""" filter_regexp = { 'Executing cache refresh triggered by request': SearchCriteria(1, True), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make sure that timers will not interfere: ar = nginx_class(cache_first_poll_delay=120, cache_poll_period=120, cache_expiration=115) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=5, line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() # Do an extra request so that we can verify that data was in fact # cached and no more than one req to mesos/marathon backends were # made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_mesos_leader_is_reresolved_by_lua( self, nginx_class, mocker, dns_server_mock, valid_user_header): # Change the TTL of `leader.mesos.` entry dns_server_mock.set_dns_entry( 'leader.mesos.', ip='127.0.0.2', ttl=self.LONG_TTL) # This should be equal or greater than 1.5 times the value of `valid=` # DNS TTL override in `resolver` config option -> 5s * 1.5 = 7.5s cache_poll_period = 8 cache_expiration = cache_poll_period - 1 cache_first_poll = 1 # We can just ask Mesos endpoints to perform request recording here, # but varying the responses of the Mesos endpoints will make tests less # prone to timing issues (i.e. `first poll` cache request will result # both mocks recording requests). mocker.send_command( endpoint_id='http://127.0.0.3:5050', func_name='enable_extra_agent', ) ar = nginx_class( cache_first_poll_delay=cache_first_poll, cache_poll_period=cache_poll_period, cache_expiration=cache_expiration, upstream_mesos="http://leader.mesos:5050", ) with GuardedSubprocess(ar): # Force cache update by issuing a request ping_mesos_agent( ar, valid_user_header, expect_status=404, agent_id=AGENT_EXTRA_ID) # Now, let's change DNS entry to point to other Mesos master dns_server_mock.set_dns_entry( 'leader.mesos.', ip='127.0.0.3', ttl=self.LONG_TTL) # Wait for cache to expire and let DNS entry be re-resolved # during the refresh time.sleep(cache_poll_period + cache_first_poll + 1) # Make sure that cache now used the right upstream ping_mesos_agent( ar, valid_user_header, expect_status=200, endpoint_id='http://127.0.0.4:15003', agent_id=AGENT_EXTRA_ID)
def test_if_first_cache_refresh_occurs_earlier(self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(1, False), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for Mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur later than usual, so that we get clear # results. ar = nginx_class(cache_poll_period=60, cache_expiration=55) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command( endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command( endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header): filter_regexp = { 'Local Mesos Master IP address is unknown, cache entry is unusable': SearchCriteria(1, True), '`Mesos Leader` state cache has been successfully updated': SearchCriteria(1, True), } ar = nginx_class(host_ip=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_occurs_regularly( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(3, False), 'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(8, True), 'Mesos state cache has been successfully updated': SearchCriteria(3, True), 'Marathon apps cache has been successfully updated': SearchCriteria(3, True), 'Marathon leader cache has been successfully updated': SearchCriteria(3, True), } cache_poll_period = 4 # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur faster than usual to speed up the tests. ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) # In total, we should get three cache updates in given time frame: timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1 with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=timeout, line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 3 assert len(marathon_requests) == 6
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `mesosstate` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class( cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=503) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header): filter_regexp = { 'Private IP address of the host is unknown, ' + 'aborting cache-entry creation for mesos leader': SearchCriteria(1, True), 'mesos leader cache has been successfully updated': SearchCriteria(1, True), } ar = nginx_class(host_ip=None) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, line_buffer=ar.stderr_line_buffer) # Just trigger the cache update: ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_cache_refresh_is_triggered_by_request( self, nginx_class, mocker, valid_user_header): """...right after Nginx has started.""" filter_regexp = { 'Executing cache refresh triggered by request': SearchCriteria(1, True), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make sure that timers will not interfere: ar = nginx_class(cache_first_poll_delay=120, cache_poll_period=120, cache_expiration=115) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=5, line_buffer=ar.stderr_line_buffer) ping_mesos_agent(ar, valid_user_header) lbf.scan_log_buffer() # Do an extra request so that we can verify that data was in fact # cached and no more than one req to mesos/marathon backends were # made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_temp_mesos_borkage_does_not_dirupt_caching( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Using stale `mesosstate` cache entry to fulfill the request': SearchCriteria(1, True), } ar = nginx_class( cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=1800, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter( filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to become stale: # cache_max_age_soft_limit + 1s for good measure time.sleep(3 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=200) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_caching_works_for_mesos_state( self, nginx_class, mocker, valid_user_header): # Enable recording for mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') ar = nginx_class() with GuardedSubprocess(ar): # Let the cache warm-up: time.sleep(CACHE_FIRST_POLL_DELAY + 1) for _ in range(3): ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') # 3 requests + only one upstream request == cache works assert len(mesos_requests) == 1
def test_if_broken_mesos_causes_mesos_cache_to_expire_and_requests_to_fail( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Cache entry `mesosstate` is too old, aborting request': SearchCriteria(1, True), } ar = nginx_class(cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=4, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to be discarded by AR: # cache_max_age_hard_limit + 1s for good measure # must be more than cache_poll_period time.sleep(4 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=503) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_temp_mesos_borkage_does_not_dirupt_caching( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Mesos state request failed: invalid response status: 500': SearchCriteria(1, False), 'Marathon apps cache has been successfully updated': SearchCriteria(2, False), 'Using stale `mesosstate` cache entry to fulfill the request': SearchCriteria(1, True), } ar = nginx_class(cache_poll_period=3, cache_expiration=2, cache_max_age_soft_limit=3, cache_max_age_hard_limit=1800, ) with GuardedSubprocess(ar): # Register Line buffer filter: lbf = LineBufferFilter(filter_regexp, timeout=5, # Just to give LBF enough time line_buffer=ar.stderr_line_buffer) # Trigger cache update using a request: ping_mesos_agent(ar, valid_user_header) # Break mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) # Wait for the cache to be old enough to become stale: # cache_max_age_soft_limit + 1s for good measure time.sleep(3 + 1) # Perform the main/test request: ping_mesos_agent(ar, valid_user_header, expect_status=200) lbf.scan_log_buffer() assert lbf.extra_matches == {}
def test_if_first_cache_refresh_occurs_earlier( self, nginx_class, mocker, valid_user_header): filter_regexp = { 'Executing cache refresh triggered by timer': SearchCriteria(1, False), 'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True), 'Mesos state cache has been successfully updated': SearchCriteria(1, True), 'Marathon apps cache has been successfully updated': SearchCriteria(1, True), 'Marathon leader cache has been successfully updated': SearchCriteria(1, True), } # Enable recording for marathon mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='record_requests') # Enable recording for Mesos mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='record_requests') # Make regular polling occur later than usual, so that we get clear # results. ar = nginx_class(cache_poll_period=60, cache_expiration=55) with GuardedSubprocess(ar): lbf = LineBufferFilter(filter_regexp, timeout=(CACHE_FIRST_POLL_DELAY + 1), line_buffer=ar.stderr_line_buffer) lbf.scan_log_buffer() # Do a request that uses cache so that we can verify that data was # in fact cached and no more than one req to mesos/marathon # backends were made ping_mesos_agent(ar, valid_user_header) mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='get_recorded_requests') marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080', func_name='get_recorded_requests') assert lbf.extra_matches == {} assert len(mesos_requests) == 1 assert len(marathon_requests) == 2
def test_if_changing_mesos_state_is_reflected_in_cache( self, nginx_class, valid_user_header, mocker): cache_poll_period = 4 ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3) with GuardedSubprocess(ar): ping_mesos_agent(ar, valid_user_header, agent_id=EXTRA_AGENT_DICT['id'], expect_status=404) mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='enable_extra_agent') # First poll (2s) + normal poll interval(4s) < 2 * normal poll # interval(4s) time.sleep(cache_poll_period * 2) ping_mesos_agent(ar, valid_user_header, agent_id=EXTRA_AGENT_DICT['id'], endpoint_id='http://127.0.0.4:15003')
def test_if_failed_request_triggered_update_is_recovered_by_timers( self, nginx_class, valid_user_header, mocker, log_catcher): # The idea here is to make Backend a bit slow, so that AR is still able # to update cache on first request. first_poll_delay = 3 poll_period = 3 cache_expiration = 2 # Take cache invalidation out of the picture ar = nginx_class( cache_first_poll_delay=first_poll_delay, cache_poll_period=poll_period, cache_expiration=cache_expiration, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) # Make mesos just a bit :) # It mus respond slower than backend_request_timeout mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) with GuardedSubprocess(ar): start = time.time() # Let's break the cache by making it update against broken Mesos: ping_mesos_agent(ar, valid_user_header, expect_status=503) time.sleep(1) # Let's make sure that the brokerage is still there ping_mesos_agent(ar, valid_user_header, expect_status=503) # Healing hands! mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=False) # Let' wait for first poll to refresh cache time.sleep(1 + (first_poll_delay - (time.time() - start))) # Verify that the cache is OK now ping_mesos_agent(ar, valid_user_header)
def test_if_failed_request_triggered_update_is_recovered_by_timers( self, nginx_class, valid_user_header, mocker, log_catcher): # The idea here is to make Backend a bit slow, so that AR is still able # to update cache on first request. first_poll_delay = 3 poll_period = 3 cache_expiration = 2 # Take cache invalidation out of the picture ar = nginx_class(cache_first_poll_delay=first_poll_delay, cache_poll_period=poll_period, cache_expiration=cache_expiration, cache_max_age_soft_limit=1200, cache_max_age_hard_limit=1800, ) # Make mesos just a bit :) # It mus respond slower than backend_request_timeout mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=True) with GuardedSubprocess(ar): start = time.time() # Let's break the cache by making it update against broken Mesos: ping_mesos_agent(ar, valid_user_header, expect_status=503) time.sleep(1) # Let's make sure that the brokerage is still there ping_mesos_agent(ar, valid_user_header, expect_status=503) # Healing hands! mocker.send_command(endpoint_id='http://127.0.0.2:5050', func_name='always_bork', aux_data=False) # Let' wait for first poll to refresh cache time.sleep(1 + (first_poll_delay - (time.time() - start))) # Verify that the cache is OK now ping_mesos_agent(ar, valid_user_header)