def testExplicitEnforcesHostUniqueness(self):
"""Should fail to make ExplicitHostGroup with duplicate hosts."""
host_list = [FakeHost('h1'), FakeHost('h2'), FakeHost('h3')]
hosts_per_spec = {
host_spec.HostSpec(['l1']): host_list[:1],
host_spec.HostSpec(['l2']): host_list
}
self.assertRaises(ValueError, host_spec.ExplicitHostGroup,
hosts_per_spec)
def testCanConstructExplicit(self):
"""Should be able to make an ExplicitHostGroup."""
host_list = [FakeHost('h1'), FakeHost('h2'), FakeHost('h3')]
hosts_per_spec = {
host_spec.HostSpec(['l1']): host_list[:1],
host_spec.HostSpec(['l2']): host_list[1:]
}
group = host_spec.ExplicitHostGroup(hosts_per_spec)
for host in host_list:
self.assertTrue(host.hostname in group.as_args()['hosts'])
def testExplicitOneHostEnoughToSatisfySpecs(self):
"""One host is enough to satisfy a HostSpec in ExplicitHostGroup."""
satisfiable_spec = host_spec.HostSpec(['l1'])
group = host_spec.ExplicitHostGroup()
group.add_host_for_spec(satisfiable_spec, FakeHost('h1'))
group.add_host_for_spec(satisfiable_spec, None)
self.assertTrue(satisfiable_spec not in group.unsatisfied_specs)
group = host_spec.ExplicitHostGroup()
group.add_host_for_spec(satisfiable_spec, None)
group.add_host_for_spec(satisfiable_spec, FakeHost('h1'))
self.assertTrue(satisfiable_spec not in group.unsatisfied_specs)
def testExplicitCanTrackSuccess(self):
"""Track success/failure in an ExplicitHostGroup."""
host_list = [FakeHost('h1'), FakeHost('h2'), FakeHost('h3')]
specs = [host_spec.HostSpec(['l1']), host_spec.HostSpec(['l2'])]
hosts_per_spec = {specs[0]: host_list[:1], specs[1]: host_list[1:]}
group = host_spec.ExplicitHostGroup(hosts_per_spec)
# Reimage just the one host that satisfies specs[0].
group.mark_host_success(host_list[0].hostname)
self.assertTrue(group.enough_hosts_succeeded())
self.assertTrue(specs[1] in group.doomed_specs)
# Reimage some host that satisfies specs[1].
group.mark_host_success(host_list[2].hostname)
self.assertTrue(group.enough_hosts_succeeded())
self.assertFalse(group.doomed_specs)
def testWaitForSingleJobHostsWithTimeout(self):
"""Discover a single host for this job then timeout."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job = FakeJob(7, hostnames=[None, None])
time.sleep(mox.IgnoreArg()).MultipleTimes()
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
self.expect_hosts_query_and_lock([job], manager, [], False)
# First, only one test in the job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
job.hostnames = [None] + expected_hostnames[1:]
self.expect_hosts_query_and_lock([job], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:])
# The second test gets a host assigned, but it's not yet running.
# Since no new running hosts are found, no locking should happen.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
job.hostnames = expected_hostnames
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:],
False)
# A timeout occurs, and only the locked hosts should be returned.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(True)
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job).AndReturn(expected_hostnames)
self.mox.ReplayAll()
# Because of the timeout only one host is returned.
expect_timeout_hostnames = ['host0']
self.assertEquals(
sorted(expect_timeout_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(
self.afe, [job],
manager,
wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS)))
def testNotIncorrectlyLocked(self):
"""Should accept hosts locked by the infrastructure."""
infra_user = '******'
self.mox.StubOutWithMock(tools, 'infrastructure_user')
tools.infrastructure_user().AndReturn(infra_user)
self.mox.ReplayAll()
host = FakeHost(locked=True, locked_by=infra_user)
self.assertFalse(tools.incorrectly_locked(host))
def testExplicitSubsetSpecSatisfiedIfAnyAre(self):
"""Ensures that any satisfied spec also satisfies a subset HostSpec."""
specs = [
host_spec.HostSpec(['l1'], ['l3']),
host_spec.HostSpec(['l1'], ['l3', 'l4']),
host_spec.HostSpec(['l1'], ['l5', 'l4']),
host_spec.HostSpec(['l1'], ['l2', 'l3', 'l4'])
]
group = host_spec.ExplicitHostGroup()
group.add_host_for_spec(specs[0], None)
group.add_host_for_spec(specs[1], FakeHost('h1'))
group.add_host_for_spec(specs[2], FakeHost('h2'))
group.add_host_for_spec(specs[3], None)
self.assertTrue(specs[0] not in group.unsatisfied_specs)
self.assertTrue(specs[1] not in group.unsatisfied_specs)
self.assertTrue(specs[2] not in group.unsatisfied_specs)
self.assertTrue(specs[3] in group.unsatisfied_specs)
def testExplicitCanTrackUnsatisfiedSpecs(self):
"""Track unsatisfiable HostSpecs in ExplicitHostGroup."""
group = host_spec.ExplicitHostGroup()
satisfiable_spec = host_spec.HostSpec(['l2'])
unsatisfiable_spec = host_spec.HostSpec(['l1'], ['e1'])
group.add_host_for_spec(unsatisfiable_spec, None)
group.add_host_for_spec(satisfiable_spec, FakeHost('h1'))
self.assertTrue(unsatisfiable_spec in group.unsatisfied_specs)
self.assertTrue(satisfiable_spec not in group.unsatisfied_specs)
def testExplicitCanTrackSuccessWithSupersets(self):
"""Track success/failure in an ExplicitHostGroup with supersets."""
host_list = [FakeHost('h1'), FakeHost('h2'), FakeHost('h3')]
specs = [
host_spec.HostSpec(['l1']),
host_spec.HostSpec(['l2']),
host_spec.HostSpec(['l2', 'l1'])
]
hosts_per_spec = {
specs[0]: host_list[:1],
specs[1]: host_list[1:2],
specs[2]: host_list[2:]
}
group = host_spec.ExplicitHostGroup(hosts_per_spec)
# Reimage just the one host that satisfies specs[2].
# Because satisfying specs[2] statisfies all the specs, we should have
# no doomed specs.
group.mark_host_success(host_list[2].hostname)
self.assertTrue(group.enough_hosts_succeeded())
self.assertFalse(group.doomed_specs)
def testWaitForMultiJobHostsToRunAndGetLocked(self):
"""Ensure we lock all running hosts for all jobs as discovered."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0', 'host2']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job0 = FakeJob(0, hostnames=[])
job1 = FakeJob(1, hostnames=[])
time.sleep(mox.IgnoreArg()).MultipleTimes()
# First, only one test in either job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job0.hostnames = [None, expected_hostnames[2]]
job1.hostnames = [None]
self.expect_hosts_query_and_lock([job0, job1], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[2:])
# The test in the second job gets a host assigned, but it's not yet
# running.
# Since no new running hosts are found, no locking should happen.
job1.hostnames = expected_hostnames[1:2]
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[2:], False)
# The second job's test's host starts running as well.
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[1:])
# All three hosts across both jobs are now running.
job0.hostnames = [expected_hostnames[0], expected_hostnames[2]]
self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts)
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job0).AndReturn(job0.hostnames)
job_status.gather_job_hostnames(mox.IgnoreArg(),
job1).AndReturn(job1.hostnames)
self.mox.ReplayAll()
self.assertEquals(
sorted(expected_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(self.afe, [job0, job1],
manager)))
def testWaitForAndLockWithTimeOutInStartJobs(self):
"""If we experience a timeout, no locked hosts are returned"""
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout')
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(True)
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job = FakeJob(7, hostnames=[None, None])
job_status.gather_job_hostnames(mox.IgnoreArg(),
job).AndReturn(expected_hostnames)
self.mox.ReplayAll()
self.assertFalse(
job_status.wait_for_and_lock_job_hosts(
self.afe, [job],
manager,
wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS))
def testWaitForSingleJobHostsToRunAndGetLockedSerially(self):
"""Lock running hosts as discovered, serially."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job = FakeJob(7, hostnames=[None, None])
time.sleep(mox.IgnoreArg()).MultipleTimes()
self.expect_hosts_query_and_lock([job], manager, [], False)
# First, only one test in the job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job.hostnames = [None] + expected_hostnames[1:]
self.expect_hosts_query_and_lock([job], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:])
# The second test gets a host assigned, but it's not yet running.
# Since no new running hosts are found, no locking should happen.
job.hostnames = expected_hostnames
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:],
False)
# The second test's host starts running as well, and the first stops.
self.expect_hosts_query_and_lock([job], manager, expected_hosts[:1])
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job).AndReturn(expected_hostnames)
self.mox.ReplayAll()
self.assertEquals(
sorted(expected_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(self.afe, [job],
manager)))
def testIncorrectlyLocked(self):
"""Should detect hosts locked by random users."""
host = FakeHost(locked=True, locked_by='some guy')
self.assertTrue(tools.incorrectly_locked(host))