def local_get_response(self):
""" Local rdb.get_response handler."""
if not (self.cache.hits == 2 and self.cache.misses == 1):
raise AssertionError('The first request should have populated '
'the cache for the others.')
default_job_params = test_utils.get_default_job_params()
lines = get_line_with_labels(
default_job_params['deps'],
self.cache._cache_backend._cache.values())
if len(lines) > 1:
raise AssertionError('Should only be one cache line left.')
# Make sure that all the jobs got different hosts, and that
# the host cached isn't being used by a job.
cached_host = lines[0].pop()
cached_host.lease()
job_hosts = []
default_job_params = test_utils.get_default_job_params()
for i in range(0, 3):
default_job_params['priority'] = i
hosts = get_hosts_for_request(self.response_map,
**default_job_params)
assert (len(hosts) == 1)
host = hosts[0]
assert (host.id not in job_hosts and cached_host.id != host.id)
return test_utils.wire_format_response_map(self.response_map)
def testCachingBasic(self):
"""Test that different requests will hit the database."""
# r1 should cache h2 and use h1; r2 should cach [] and use h2
# at the end the cache should contain one stale line, with
# h2 in it, and one empty line since r2 acquired h2.
default_params = test_utils.get_default_job_params()
self.create_job(**default_params)
default_params['deps'] = default_params['deps'][0]
self.create_job(**default_params)
for i in range(0, 2):
self.db_helper.create_host('h%s' % i,
**test_utils.get_default_host_params())
queue_entries = self._dispatcher._refresh_pending_queue_entries()
def local_get_response(self):
""" Local rdb.get_response handler."""
requests = self.response_map.keys()
if not (self.cache.hits == 0 and self.cache.misses == 2):
raise AssertionError(
'Neither request should have hit the '
'cache, but both should have inserted into it.')
lines = get_line_with_labels(
test_utils.DEFAULT_DEPS,
self.cache._cache_backend._cache.values())
if len(lines) > 1:
raise AssertionError(
'Caching was too agressive, '
'the second request should not have cached anything '
'because it used the one free host.')
cached_host = lines[0].pop()
default_params = test_utils.get_default_job_params()
job1_host = get_hosts_for_request(self.response_map,
**default_params)[0]
default_params['deps'] = default_params['deps'][0]
job2_host = get_hosts_for_request(self.response_map,
**default_params)[0]
if (job2_host.hostname == job1_host.hostname
or cached_host.hostname
not in [job2_host.hostname, job1_host.hostname]):
raise AssertionError(
'Wrong host cached %s. The first job '
'should have cached the host used by the second.' %
cached_host.hostname)
# Shouldn't be able to lease this host since r2 used it.
try:
cached_host.lease()
except rdb_utils.RDBException:
pass
else:
raise AssertionError('Was able to lease a stale host. The '
'second request should have leased it.')
return test_utils.wire_format_response_map(self.response_map)
self.god.stub_with(rdb.AvailableHostRequestHandler, 'get_response',
local_get_response)
self.check_hosts(rdb_lib.acquire_hosts(queue_entries))
def testCachingEmptyList(self):
"""Test that the 'no available hosts' condition isn't a cache miss."""
default_params = test_utils.get_default_job_params()
for i in range(0, 3):
default_params['parent_job_id'] = i
self.create_job(**default_params)
default_host_params = test_utils.get_default_host_params()
self.db_helper.create_host('h1', **default_host_params)
def local_get_response(self):
""" Local rdb.get_response handler."""
if not (self.cache.misses == 1 and self.cache.hits == 2):
raise AssertionError(
'The first request should have taken h1 '
'while the other 2 should have hit the cache.')
request = test_utils.AbstractBaseRDBTester.get_request(
test_utils.DEFAULT_DEPS, test_utils.DEFAULT_ACLS)
key = self.cache.get_key(deps=request.deps, acls=request.acls)
if self.cache._cache_backend.get(key) != []:
raise AssertionError('A request with no hosts does not get '
'cached corrrectly.')
return test_utils.wire_format_response_map(self.response_map)
queue_entries = self._dispatcher._refresh_pending_queue_entries()
self.god.stub_with(rdb.AvailableHostRequestHandler, 'get_response',
local_get_response)
self.check_hosts(rdb_lib.acquire_hosts(queue_entries))
def local_get_response(self):
""" Local rdb.get_response handler."""
default_job_params = test_utils.get_default_job_params()
# Confirm that even though the third job hit the cache, it wasn't
# able to use the cached host because it was already leased, and
# that it doesn't add it back to the cache.
assert (self.cache.misses == 2 and self.cache.hits == 1)
lines = get_line_with_labels(
default_job_params['deps'],
self.cache._cache_backend._cache.values())
assert (len(lines) == 0)
assert (int(self.cache.mean_staleness()) == 100)
return test_utils.wire_format_response_map(self.response_map)
def testCachingPriority(self):
"""Test requests with the same deps but different priorities."""
# All 3 jobs should find hosts, and there should be one host left
# behind in the cache. The first job will take one host and cache 3,
# the second will take one and cache 2, while the last will take one.
# The remaining host in the cache should not be stale.
default_job_params = test_utils.get_default_job_params()
for i in range(0, 3):
default_job_params['priority'] = i
job = self.create_job(**default_job_params)
default_host_params = test_utils.get_default_host_params()
for i in range(0, 4):
self.db_helper.create_host('h%s' % i, **default_host_params)
queue_entries = self._dispatcher._refresh_pending_queue_entries()
def local_get_response(self):
""" Local rdb.get_response handler."""
if not (self.cache.hits == 2 and self.cache.misses == 1):
raise AssertionError('The first request should have populated '
'the cache for the others.')
default_job_params = test_utils.get_default_job_params()
lines = get_line_with_labels(
default_job_params['deps'],
self.cache._cache_backend._cache.values())
if len(lines) > 1:
raise AssertionError('Should only be one cache line left.')
# Make sure that all the jobs got different hosts, and that
# the host cached isn't being used by a job.
cached_host = lines[0].pop()
cached_host.lease()
job_hosts = []
default_job_params = test_utils.get_default_job_params()
for i in range(0, 3):
default_job_params['priority'] = i
hosts = get_hosts_for_request(self.response_map,
**default_job_params)
assert (len(hosts) == 1)
host = hosts[0]
assert (host.id not in job_hosts and cached_host.id != host.id)
return test_utils.wire_format_response_map(self.response_map)
self.god.stub_with(rdb.AvailableHostRequestHandler, 'get_response',
local_get_response)
self.check_hosts(rdb_lib.acquire_hosts(queue_entries))
def testStaleCacheLine(self):
"""Test that a stale cache line doesn't satisfy a request."""
# Create 3 jobs, all of which can use the same hosts. The first
# will cache the only remaining host after taking one, the second
# will also take a host, but not cache anything, while the third
# will try to use the host cached by the first job but fail because
# it is already leased.
default_params = test_utils.get_default_job_params()
default_params['priority'] = 2
self.create_job(**default_params)
default_params['priority'] = 1
default_params['deps'] = default_params['deps'][0]
self.create_job(**default_params)
default_params['priority'] = 0
default_params['deps'] = test_utils.DEFAULT_DEPS
self.create_job(**default_params)
host_1 = self.db_helper.create_host(
'h1', **test_utils.get_default_host_params())
host_2 = self.db_helper.create_host(
'h2', **test_utils.get_default_host_params())
queue_entries = self._dispatcher._refresh_pending_queue_entries()
def local_get_response(self):
""" Local rdb.get_response handler."""
default_job_params = test_utils.get_default_job_params()
# Confirm that even though the third job hit the cache, it wasn't
# able to use the cached host because it was already leased, and
# that it doesn't add it back to the cache.
assert (self.cache.misses == 2 and self.cache.hits == 1)
lines = get_line_with_labels(
default_job_params['deps'],
self.cache._cache_backend._cache.values())
assert (len(lines) == 0)
assert (int(self.cache.mean_staleness()) == 100)
return test_utils.wire_format_response_map(self.response_map)
self.god.stub_with(rdb.AvailableHostRequestHandler, 'get_response',
local_get_response)
acquired_hosts = list(rdb_lib.acquire_hosts(queue_entries))
self.assertTrue(acquired_hosts[0].id == host_1.id
and acquired_hosts[1].id == host_2.id
and acquired_hosts[2] is None)
def local_get_response(self):
""" Local rdb.get_response handler."""
requests = self.response_map.keys()
if not (self.cache.hits == 0 and self.cache.misses == 2):
raise AssertionError(
'Neither request should have hit the '
'cache, but both should have inserted into it.')
lines = get_line_with_labels(
test_utils.DEFAULT_DEPS,
self.cache._cache_backend._cache.values())
if len(lines) > 1:
raise AssertionError(
'Caching was too agressive, '
'the second request should not have cached anything '
'because it used the one free host.')
cached_host = lines[0].pop()
default_params = test_utils.get_default_job_params()
job1_host = get_hosts_for_request(self.response_map,
**default_params)[0]
default_params['deps'] = default_params['deps'][0]
job2_host = get_hosts_for_request(self.response_map,
**default_params)[0]
if (job2_host.hostname == job1_host.hostname
or cached_host.hostname
not in [job2_host.hostname, job1_host.hostname]):
raise AssertionError(
'Wrong host cached %s. The first job '
'should have cached the host used by the second.' %
cached_host.hostname)
# Shouldn't be able to lease this host since r2 used it.
try:
cached_host.lease()
except rdb_utils.RDBException:
pass
else:
raise AssertionError('Was able to lease a stale host. The '
'second request should have leased it.')
return test_utils.wire_format_response_map(self.response_map)
def testDummyCache(self):
"""Test that the dummy cache doesn't save hosts."""
# Create 2 jobs and 3 hosts. Both the jobs should not hit the cache,
# nor should they cache anything, but both jobs should acquire hosts.
default_params = test_utils.get_default_job_params()
default_host_params = test_utils.get_default_host_params()
for i in range(0, 2):
default_params['parent_job_id'] = i
self.create_job(**default_params)
self.db_helper.create_host('h%s' % i, **default_host_params)
self.db_helper.create_host('h2', **default_host_params)
queue_entries = self._dispatcher._refresh_pending_queue_entries()
self.god.stub_with(rdb_cache_manager.RDBHostCacheManager, 'use_cache',
False)
def local_get_response(self):
""" Local rdb.get_response handler."""
requests = self.response_map.keys()
if not (self.cache.hits == 0 and self.cache.misses == 2):
raise AssertionError(
'Neither request should have hit the '
'cache, but both should have inserted into it.')
# Make sure both requests actually found a host
default_params = test_utils.get_default_job_params()
job1_host = get_hosts_for_request(self.response_map,
**default_params)[0]
default_params['parent_job_id'] = 1
job2_host = get_hosts_for_request(self.response_map,
**default_params)[0]
if (not job1_host or not job2_host
or job2_host.hostname == job1_host.hostname):
raise AssertionError('Excected acquisitions did not occur.')
assert (hasattr(self.cache._cache_backend, '_cache') == False)
return test_utils.wire_format_response_map(self.response_map)
self.god.stub_with(rdb.AvailableHostRequestHandler, 'get_response',
local_get_response)
self.check_hosts(rdb_lib.acquire_hosts(queue_entries))
def local_get_response(self):
""" Local rdb.get_response handler."""
requests = self.response_map.keys()
if not (self.cache.hits == 0 and self.cache.misses == 2):
raise AssertionError(
'Neither request should have hit the '
'cache, but both should have inserted into it.')
# Make sure both requests actually found a host
default_params = test_utils.get_default_job_params()
job1_host = get_hosts_for_request(self.response_map,
**default_params)[0]
default_params['parent_job_id'] = 1
job2_host = get_hosts_for_request(self.response_map,
**default_params)[0]
if (not job1_host or not job2_host
or job2_host.hostname == job1_host.hostname):
raise AssertionError('Excected acquisitions did not occur.')
assert (hasattr(self.cache._cache_backend, '_cache') == False)
return test_utils.wire_format_response_map(self.response_map)
